cfengine · craigcomstock · Aug 21, 2025 · Aug 19, 2025 · Aug 19, 2025 · Aug 19, 2025
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -0,0 +1,125 @@
+pipeline {
+  agent { label 'CONTAINERS' }
+  environment {
+    REPOS = "core enterprise nova masterfiles northerntechhq/nt-docs"
+    PR_BASE="${pullRequest.base}"
+    PACKAGE_JOB = "cf-remote"
+    PACKAGE_UPLOAD_DIRECTORY = "n/a"
+    PACKAGE_BUILD = "n/a"
+  }
+  parameters {
+    string(name: "CORE_REV", defaultValue: '', description: 'used for changelog, examples. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "NOVA_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "ENTERPRISE_REV", defaultValue: '', description: 'used for changelog. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "MASTERFILES_REV", defaultValue: '', description: 'used to document masterfiles. Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "NT_DOCS_REV", defaultValue: '', description: 'Use NUMBER or "pull/NUMBER/merge" for pull request (it\'s merged version, THIS DOESN\'T MERGE THE PR) or "pull/NUMBER/head" to build the docs with the non-merged code. Special syntax \'tag:SOME_TAG\' can be used to use a tag as a revision.')
+    string(name: "DOCS_BRANCH", defaultValue: '', description: 'Where to upload artifacts - to http://buildcache.cloud.cfengine.com/packages/build-documentation-$DOCS_BRANCH/ and https://docs.cfengine.com/docs/$DOCS_BRANCH/')
+    string(name: "PACKAGE_JOB", defaultValue: 'cf-remote', description: 'where to get CFEngine HUB package from, either a dir at http://buildcache.cloud.cfengine.com/packages like testing-pr or a keyword cf-remote to use cf-remote download')
+    string(name: "USE_NIGHTLIES_FOR", defaultValue: '', description: 'branch whose nightlies to use (master, 3.18.x, etc) - will be one of http://buildcache.cloud.cfengine.com/packages/testing-pr/jenkins-$USE_NIGHTLIES_FOR-nightly-pipeline-$NUMBER/')
+  }
+  options {
+    checkoutToSubdirectory('documentation')
+  }
+  stages {
+    stage('Environment check') {
+      steps {
+        sh 'env'
+        sh 'whoami; pwd; ls'
+        sh 'uname -a; cat /etc/os-release'
+      }
+    }
+    // we clean FIRST and NOT at the end of the job so that we can replay various stages and have the build result from previous runs
+    stage('Clean workspace') {
+      steps {
+        sh 'for r in $REPOS; do rm -rf "$(basename "$r")"; done'
+      }
+    }
+    stage('Checkout repositories'){
+      steps {
+        sh "echo \"${pullRequest.title}\" > pull-request-title"
+        sh "echo \"${pullRequest.body}\" > pull-request-body"
+// TODO: change URL after merging https://github.com/cfengine/buildscripts/pull/1848
+        sh "curl -O https://raw.githubusercontent.com/craigcomstock/buildscripts/refs/heads/ENT-12581/ci/create-revisions-file.sh"
+        sh "chmod u+x ./create-revisions-file.sh"
+        sh "./create-revisions-file.sh"
+        sh "cat revisions"
+        sh "curl -O https://gitlab.com/Northern.tech/OpenSource/GODS/-/raw/master/parallel_git_rev_fetch.sh"
+        sh "chmod u+x ./parallel_git_rev_fetch.sh"
+
+        withCredentials([sshUserPrivateKey(credentialsId:"autobuild", keyFileVariable: "key")]) {
+          sh 'export GIT_SSH_COMMAND="ssh -i $key"; ./parallel_git_rev_fetch.sh revisions'
+        }
+      }
+    }
+    stage('Build documentation') {
+      steps {
+        sh 'bash -x documentation/generator/build/run.sh'
+      }
+    }
+    stage('Publish to buildcache') {
+      steps {
+        sshPublisher(
+          // we must use alwaysPublishFromMaster: true because our CONTAINERS build hosts are not in the private network which has access to buildcache.cloud.cfengine.com
+          alwaysPublishFromMaster: true,
+          publishers: [
+            sshPublisherDesc(
+              configName: 'buildcache.cloud.cfengine.com',
+              transfers: [
+                sshTransfer(
+                  cleanRemote: false,
+                  excludes: '',
+                  execCommand: '''
+#!/usr/bin/env bash
+set -x
+WRKDIR="$(pwd)"
+export WRKDIR
+
+mkdir -p upload
+mkdir -p output
+
+# find two tarballs
+archive="$(find upload -name "cfengine-documentation-*.tar.gz")"
+tarball=$(findfind upload -name packed-for-shipping.tar.gz)
+echo "TARBALL: $tarball"
+echo "ARCHIVE: $archive"
+
+# unpack $tarball
+( # shubshell to change directories
+  cd "$(dirname "$tarball")" || exit
+  tar zxvf packed-for-shipping.tar.gz
+  rm packed-for-shipping.tar.gz
+
+  # move $archive to the _site
+  mv "$WRKDIR/$archive" _site
+
+  ls -la
+)
+
+ls -la upload
+
+# note: this triggers systemd job to AV-scan new files
+# and move them to proper places
+mv upload/* output
+''',
+                  execTimeout: 120000,
+                  flatten: false,
+                  makeEmptyDirs: false,
+                  noDefaultExcludes: false,
+                  patternSeparator: '[, ]+',
+                  remoteDirectory: 'upload/$BUILD_TAG/build-documentation-$DOCS_BRANCH/$BUILD_TAG/',
+                  remoteDirectorySDF: false,
+                  removePrefix: '',
+                  sourceFiles: 'output/'
+                ) // sshTransfer
+               ], // transfers
+               usePromotionTimestamp: false,
+               useWorkspaceInPromotion: false,
+               verbose: false
+             ) // sshPublisherDesc
+           ] // publishers
+         ) // sshPublisher
+      } // steps
+    } // stage('Publish to buildcache')
+  } // stages
+}
diff --git a/generator/build/main.sh b/generator/build/main.sh
@@ -9,13 +9,22 @@ if [ "$#" != 4 ]; then
     exit 1
 fi
 
+echo "$(basename "$0"): Diagnostic facts about execution environment:"
+echo "======"
+whoami
+cat /etc/os-release
+free -h
+df -h
+uname -a
+echo "======"
+
+
 export BRANCH=$1
 export PACKAGE_JOB=$2
 export PACKAGE_UPLOAD_DIRECTORY=$3
 export PACKAGE_BUILD=$4
 
 export JOB_TO_UPLOAD=$PACKAGE_JOB
-export FLAG_FILE_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/PACKAGES_HUB_x86_64_linux_ubuntu_22/core-commitID"
 export NO_OUTPUT_DIR=1
 
 env
@@ -64,30 +73,37 @@ test ! -z "$PACKAGE_UPLOAD_DIRECTORY"
 test ! -z "$PACKAGE_BUILD"
 
 
-echo "Waiting for flag file to appear"
-for i in $(seq 30); do
-    if wget -O- "$FLAG_FILE_URL"; then
-      break
-    fi
-    echo "Waiting 10 sec"
-    sleep 10
-done
-# check if flag file is there - if not, script will fail here
-wget -O- "$FLAG_FILE_URL"
-
-echo "Detecting version"
-HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22
-HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/"
-HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')"
 
-fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12
+echo "Install hub package"
+if [ "$PACKAGE_JOB" = "cf-remote" ]; then
+  echo "Install using cf-remote"
+  sudo apt update -y
+  sudo apt install -y python3-venv pipx
+  pipx install cf-remote
+  export PATH="$HOME/.local/bin:$PATH"
+  # shellcheck source=/dev/null
+  source /etc/os-release
+  rm -rf ~/.cfengine/cf-remote/packages # to ensure we only get one
+  cf-remote --version "$BRANCH" download "${ID}$(echo "${VERSION_ID}" | cut -d. -f1)" hub "$(uname -m)"
+  find "$HOME/.cfengine" # debug
+  find "$HOME/.cfengine" -name '*.deb' -print0 | xargs -0 -I{} cp {} cfengine-nova-hub.deb
+else
+  echo "Installing with old-style fetch_file function"
+  HUB_DIR_NAME=PACKAGES_HUB_x86_64_linux_ubuntu_22
+  HUB_DIR_URL="http://buildcache.cfengine.com/packages/$PACKAGE_JOB/$PACKAGE_UPLOAD_DIRECTORY/$HUB_DIR_NAME/"
+  HUB_PACKAGE_NAME="$(wget "$HUB_DIR_URL" -O- | sed '/\.deb/!d;s/.*"\([^"]*\.deb\)".*/\1/')"
+
+  fetch_file "$HUB_DIR_URL$HUB_PACKAGE_NAME" "cfengine-nova-hub.deb" 12
+fi
 
 sudo apt-get -y purge cfengine-nova-hub || true
 sudo rm -rf /*/cfengine
 
-# unpack
+# we unpack the hub package instead of installing to get around trouble with the package trying to start up services in a container which doesn't work all that well (yet, 2025)
 sudo dpkg --unpack cfengine-nova-hub.deb
 rm cfengine-nova-hub.deb
+
+# TODO: why copy the masterfiles from the package over the top of one we checked out which could have changes from a PR?
 sudo cp -a /var/cfengine/share/NovaBase/masterfiles "$WRKDIR"
 sudo chmod -R a+rX "$WRKDIR"/masterfiles
 

diff --git a/generator/build/run.sh b/generator/build/run.sh
@@ -14,12 +14,22 @@ fi
 # * masterfiles (used to document masterfies)
 # * documentation (this repo)
 
-# These env vars must be defined:
-true "${BRANCH?undefined}"
+# The PACKAGE* vars are not needed for fast-build jobs as they use cf-remote --version $BRANCH install
+# We still require them to have a value but by current convention (until cf-remote --version testing-pr-build-number works) we set them to cf-remote in documentation/Jenkinsfile
 true "${PACKAGE_JOB?undefined}"
 true "${PACKAGE_UPLOAD_DIRECTORY?undefined}"
 true "${PACKAGE_BUILD?undefined}"
 
+# figure out BRANCH from jenkins environment variables
+if [ -n "$PR_BASE" ]; then
+  # PR_BASE comes from documentation/Jenkinsfile and ${pullRequest.base}
+  BRANCH="$PR_BASE"
+elif [ -n "$BRANCH_NAME" ]; then
+  # jenkins, for pull requests this will be e.g. PR-<number> so not used
+  # for non-pull reqeusts this will be master, 3.24.x, etc
+  BRANCH="$BRANCH_NAME"
+fi
+
 c=$(buildah from -v "$PWD":/nt docs-revamp-22)
 trap 'buildah run "$c" bash -c "sudo chown -R root:root /nt; sudo chmod -R a+rwX /nt"; buildah rm "$c" >/dev/null' EXIT
 buildah run "$c" bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD"