Improvement for deployment script

.jenkins/Jenkinsfile

@@ -267,17 +267,7 @@ pipeline {
                         }
                     }
                     steps {
-                        sh 'mvn -B site-deploy -DskipWebpack'
-                        script {
-                            // maven site-deploy changes permission on the parent site directory (pavlab-starter-parent),
-                            // but the Gemma site directory is not a subdirectory of it.
-                            // This is replicating what the plugin is doing https://maven.apache.org/plugins/maven-site-plugin/deploy-mojo.html 
-                            sh "chmod -Rf g+w,a+rX ${params.MAVEN_SITES_DIR}/gemma/gemma-${gemmaVersion} || true"
-                            if (dataDir != null) {
-                                sh "ln -Tsf ${params.MAVEN_SITES_DIR}/gemma/gemma-${gemmaVersion} ${dataDir}/gemma-devsite"
-                                sh "ln -Tsf ${params.MAVEN_SITES_DIR}/baseCode/baseCode-${baseCodeVersion} ${dataDir}/baseCode-site"
-                            }
-                        }
+                        sh "./scripts/deploy-maven-site.sh"
                     }
                 }
                 stage('Deploy Gemma Web') {
@@ -291,7 +281,7 @@ pipeline {
                         }
                     }
                     steps {
-                        sh "env GEMMA_WEB_DEPLOY_SERVER=${deployServer} GEMMA_WEB_PREFIX=${params.DEPLOY_DIR_PREFIX} ./gemma-web/deploy.sh ${deployRef}"
+                        sh "env GEMMA_WEB_DEPLOY_SERVER=${deployServer} GEMMA_WEB_PREFIX=${params.DEPLOY_DIR_PREFIX} ./scripts/deploy-web.sh ${deployRef}"
                     }
                 }
                 stage('Deploy Gemma CLI') {
@@ -305,8 +295,8 @@ pipeline {
                         }
                     }
                     steps {
-                        sh "env GEMMA_CLI_PREFIX=${params.CLI_DIR_PREFIX} ./gemma-cli/deploy.sh ${cliRef}"
-                        sh "env GEMMA_CLI_WIKI_PAGE_SUFFIX='${cliWikiPageSuffix}' ./gemma-cli/deploy-wiki.sh 'pavlab-wiki:Global/gemma/Gemma Landing Page/Gemma Curation/List of Gemma CLI Tools${cliWikiPageSuffix}'"
+                        sh "env GEMMA_CLI_PREFIX=${params.CLI_DIR_PREFIX} ./scripts/deploy-cli.sh ${cliRef}"
+                        sh "env GEMMA_CLI_WIKI_PAGE_SUFFIX='${cliWikiPageSuffix}' ./scripts/deploy-cli-wiki.sh 'pavlab-wiki:Global/gemma/Gemma Landing Page/Gemma Curation/List of Gemma CLI Tools${cliWikiPageSuffix}'"
                     }
                 }
             }

justfile

@@ -1,14 +1,23 @@
 MAVEN := env("MAVEN", "mvn")
 MAVEN_ARGS := "-am -Prelease -DskipTests"
 
-export GEMMA_CLI_DEPLOY_SERVER := "pavlab"
+# This is the general deployment server
+DEPLOY_SERVER := "frink.pavlab.msl.ubc.ca"
+PRODUCTION_DEPLOY_SERVER := "moe.pavlab.msl.ubc.ca"
+DEVELOPMENT_DEPLOY_SERVER := "chalmers.pavlab.msl.ubc.ca"
+
+export GEMMA_CLI_DEPLOY_SERVER := DEPLOY_SERVER
 export GEMMA_CLI_PREFIX := "/space/opt/gemma-cli"
 export GEMMA_CLI_WIKI_PAGE_SUFFIX := " (generated)"
 export GEMMA_CLI_WIKI_DEPLOY_DEST := f"pavlab-wiki:Global/gemma/Gemma Landing Page/Gemma Curation/List of Gemma CLI Tools{{GEMMA_CLI_WIKI_PAGE_SUFFIX}}/"
 
-export GEMMA_WEB_DEPLOY_SERVER := "chalmers"
+export GEMMA_WEB_DEPLOY_SERVER := DEVELOPMENT_DEPLOY_SERVER
 export GEMMA_WEB_PREFIX := "/var/local/tomcat"
 
+export GEMMA_APPDATA_DIR := "/space/gemmaData"
+
+export MAVEN_SITES_DEPLOY_SERVER := DEPLOY_SERVER
+
 default: build
 
 build:
@@ -17,34 +26,48 @@ build:
 build-web:
 	{{MAVEN}} {{MAVEN_ARGS}} package -pl gemma-web
 
+generate-dwr-client:
+    /scripts/generate-dwr-client.py
+
 build-cli:
 	{{MAVEN}} {{MAVEN_ARGS}} package -pl gemma-cli
 
 update-completion-scripts: build-cli
 	env GEMMA_CLI_ALIAS=gemma-cli ./gemma-cli/update-completion-scripts.sh
 	env GEMMA_CLI_ALIAS=gemma-cli-staging ./gemma-cli/update-completion-scripts.sh
 
-generate-wiki: build-cli
+generate-wiki page_suffix=" (generated)": build-cli
 	./gemma-cli/target/appassembler/bin/gemma-cli --completion --completion-wiki --completion-wiki-output-dir gemma-cli/target/wiki --completion-wiki-page-suffix "{{GEMMA_CLI_WIKI_PAGE_SUFFIX}}"
 
-deploy-wiki: generate-wiki
-	./gemma-cli/deploy-wiki.sh "{{GEMMA_CLI_WIKI_DEPLOY_DEST}}"
+deploy-wiki page_suffix: (generate-wiki page_suffix)
+	env GEMMA_CLI_WIKI_PAGE_SUFFIX="{{page_suffix}}" ./scripts/deploy-cli-wiki.sh "{{GEMMA_CLI_WIKI_DEPLOY_DEST}}"
 
-deploy-wiki-staging:
-    just --justfile "{{justfile()}}" --set GEMMA_CLI_WIKI_PAGE_SUFFIX " (staging)" deploy-wiki
+[confirm('Deploying to production manually is very dangerous. Proceed?')]
+deploy-wiki-production: (deploy-wiki " (generated)")
 
-deploy-wiki-dev:
-    just --justfile "{{justfile()}}" --set GEMMA_CLI_WIKI_PAGE_SUFFIX " (development)" deploy-wiki
+deploy-wiki-staging: (deploy-wiki " (staging)")
+
+deploy-wiki-dev: (deploy-wiki " (development)")
 
 deploy-web ref: build-web
-	./gemma-web/deploy.sh "{{ref}}"
+	./scripts/deploy-web.sh "{{ref}}"
+
+[confirm('Deploying to production manually is very dangerous. Proceed?')]
+deploy-web-production:
+    just --justfile "{{justfile()}}" --set GEMMA_WEB_DEPLOY_SERVER {{PRODUCTION_DEPLOY_SERVER}} deploy-web 'gemma'
 
 deploy-web-staging: (deploy-web 'gemma-staging')
 
 deploy-web-dev: (deploy-web 'gemma')
 
-deploy-cli ref: build-cli update-completion-scripts
-	./gemma-cli/deploy.sh "{{ref}}"
+deploy-maven-site:
+    ./deploy-maven-site.sh
+
+deploy-cli ref: build-cli
+	./scripts/deploy-cli.sh "{{ref}}"
+
+[confirm('Deploying to production manually is very dangerous. Proceed?')]
+deploy-cli-production: (deploy-cli 'production')
 
 deploy-cli-staging: (deploy-cli 'staging')
 
@@ -56,3 +79,18 @@ clean-cli:
 
 clean-web:
 	{{MAVEN}} {{MAVEN_ARGS}} clean -pl gemma-web
+
+hotfix-start:
+    {{MAVEN}} gitflow:hotfix-start
+
+hotfix-finish:
+    {{MAVEN}} gitflow:hotfix-finish
+
+release-start:
+    {{MAVEN}} gitflow:release-start
+
+release-finish:
+    {{MAVEN}} gitflow:release-finish
+
+support-start:
+    {{MAVEN}} gitflow:support-start

gemma-cli/deploy-wiki.sh → scripts/deploy-cli-wiki.sh

@@ -53,7 +53,7 @@ if [ ! -d "$gemma_cli_dir" ]; then
 fi
 
 echo "Generating Gemma CLI Wiki pages under $gemma_cli_wiki_dir..."
-./gemma-cli/target/appassembler/bin/gemma-cli --completion --completion-wiki --completion-wiki-output-dir "$gemma_cli_wiki_dir" --completion-wiki-page-suffix "$gemma_cli_wiki_page_suffix"
+"$gemma_cli_dir/bin/gemma-cli --completion --completion-wiki --completion-wiki-output-dir "$gemma_cli_wiki_dir" --completion-wiki-page-suffix "$gemma_cli_wiki_page_suffix"
 
 echo "Deploying Gemma CLI Wiki to $wiki_dest..."
 rclone "$rclone_command" "$gemma_cli_wiki_dir/List of Gemma CLI Tools$gemma_cli_wiki_page_suffix/" "$wiki_dest" --exclude '@*/**' --exclude '*.url'

gemma-cli/deploy.sh → scripts/deploy-cli.sh

@@ -13,6 +13,8 @@
 # GEMMA_CLI_DEPLOY_SERVER: The server to deploy the Gemma CLI to.
 # GEMMA_CLI_PREFIX:        The prefix for the Gemma CLI deployment directory (defaults to /space/opt/gemma-cli).
 #
+# Note: you need a recent version of rsync with the --mkpath option.
+#
 
 set -e
 
@@ -78,8 +80,10 @@ if [ -z "$deploy_server" ]; then
 	mkdir -p "$gemma_cli_prefix/refs"
 	ln -sTf "$build_dir" "$build_ref_dir"
 else
-	ssh "$deploy_server" mkdir -p "$gemma_cli_prefix/refs"
-	ssh "$deploy_server" ln -sTf "$build_dir" "$build_ref_dir"
+  ssh "$deploy_server" <EOF
+  mkdir -p "$gemma_cli_prefix/refs"
+  ln -sTf "$build_dir" "$build_ref_dir"
+  EOF
 fi
 
 echo "Deployment completed!"

scripts/deploy-maven-site.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+#
+# Environment variables:
+# MAVEN:               Executable to use for Maven commands (defaults to mvn).
+# GEMMA_APPDATA_HOME:  Directory where Gemma data is stored on the deployment server. This will be used to create
+#                      symbolic links to specific Maven websites that are then accessible at https://gemma.msl.ubc.ca/resources/.
+#
+
+MAVEN=mvn
+
+# TODO: check for local deployment
+if "$($MAVEN help:evaluate -Dexpression=local-deploy -q -DforceStdout)" ]; then
+  deploy_server=
+else
+  deploy_server=$($MAVEN help:evaluate -Dexpression=pavlab.server -q -DforceStdout)
+fi
+maven_site_dir=$($MAVEN help:evaluate -Dexpression=pavlab.siteDir -q -DforceStdout)
+gemma_version=$($MAVEN help:evaluate -Dexpression=project.version -q -DforceStdout)
+basecode_version=$($MAVEN help:evaluate -Dartifact=baseCode:baseCode -Dexpression=project.version -q -DforceStdout)
+
+if [ -n "$deploy_server" ]; then
+  echo "Deploying Maven site to $deploy_server:$maven_site_dir..."
+else
+  echo "Deploying Maven site to $maven_site_dir..."
+fi
+mvn -B site-deploy -DskipWebpack
+([ -n "$deploy_server" ] && ssh "$deploy_server" || eval)  <EOF
+  # maven site-deploy changes permission on the parent site directory (pavlab-starter-parent),
+  # but the Gemma site directory is not a subdirectory of it.
+  # This is replicating what the plugin is doing https://maven.apache.org/plugins/maven-site-plugin/deploy-mojo.html
+  chmod -Rf g+w,a+rX "$maven_site_dir/gemma/gemma-$gemma_version" || true
+
+  # create symbolic links
+  if [ -n "$GEMMA_APPDATA_HOME" ]; then
+    echo "Creating symbolic links for Gemma and baseCode in $GEMMA_APPDATA_HOME..."
+    mkdir -p "$GEMMA_APPDATA_HOME"
+    ln -Tsf "$maven_site_dir/gemma/gemma-$gemma_version" "$GEMMA_APPDATA_HOME/gemma-devsite"
+    ln -Tsf "$maven_site_dir/baseCode/baseCode-$basecode_version" "$GEMMA_APPDATA_HOME/baseCode-site"
+  fi
+EOF

gemma-web/deploy.sh → scripts/deploy-web.sh

@@ -12,6 +12,8 @@
 # GEMMA_WEB_PREFIX:        The prefix for the Gemma Web deployment directory (defaults to /var/local/tomcat). Note that
 #                          the actual installation is done under /var/local/tomcat/$ref.
 #
+# Note: you need a recent version of rsync with the --mkpath option.
+#
 
 set -e