newt-machine.yaml

#cloud-config
#
# newt-init.yaml provides a development environemt for building
# web applications based on PostgreSQL 16, PostgREST 12, and Newt.
# It also include Python 3 for web service development and 
# generates a "ws" command using and using Python 3's http.server
# to emulate a front end web server.
#
# Two full text search engine options are also available. Both
# can function as JSON data sources in a Newt application.
#
#   Solr 9.4
#   Opensearch 2.11.1
#
# This cloud init target Ubuntu's Multipass and assume the image
# available is Jammy (22.04 LTS). Here's an example creating a VM
# with Multipass call "newt-machine" and accessing it via Multipass's
# shell.
#
# You need more than the default disk image size for fit all the stuff.
# When you initialize your VM use the following command
#
# ~~~
# multipass launch --name newt-machine \
#                  --memory 4G \
#                  --disk 250G \
#                  --cloud-init newt-init.yaml
# ~~~
#
# When you're done you can stop the VM with
#
# ~~~
# multipass stop newt-machine
# ~~~
#
# You can remove newt-machine completely with
#
# ~~~
# multipass stop newt-machine
# multipass delete newt-machine
# multipass purge newt-machine
# ~~~
#
# The first time you run your VM you will need to run the following
# additional setup bash scripts. Then you can choose to install
# to install Haskell (GHCup) so you can compile the latest Pandoc and PostgREST
# and Opensearch/Opensearch Dashboard.
#
# 1. 01-setup-scripts.bash, creates "$HOME/bin" and "$HOME/sbin" and adds things ito your .bashrc
# 2. 02-add-python-packages.bash, adds commonly used Python packages to the system
# 3. 03-add-go-and-caltechlibrary-tools.bash, builds and install DLD Go based tools
# 4. 04-ghcup-install.bash, installs Haskell toolchain and manager
# 5. 05-pandoc-build.bash, downloads the latest Pandoc and builds it
# 6. 06-postgrest-build.bash, downloads the latest PostgREST and builds it.
# 7. 07-add-newt.bash, downloads the Opensearch/Opensearch Dashbaord debs and installs them
# 8. 08-add-newt.bash, downloads Solr 9.5.0 and installs it from a tar ball.
#
write_files:
  - content: |
      #!/bin/bash
      cat <<EOT
    
      Weclome to the Newt Machine. You can
      access if with
    
          multipass shell newt-machine
    
      You can then run one or more additional
      configuration scripts to add additional
      software for development.
    
        00-menu-of-scripts.bash (displays this help page)
        01-setup-scripts.bash
        02-add-python-packages.bash
        03-add-go-and-caltechlibrary-tools.bash
        04-ghcup-install.bash
        05-pandoc-build.bash
        06-postgrest-build.bash
        07-add-opensearch.bash
        08-add-solr.bash
    
      The first and second ones are needed for
      Newt Projects. The rest are optional
    
      You can grant yourself SSH access with the following
      command when you connect using multipass shell.
    
         ssh-keygen
             curl -L -o - https://github.com/${USER}.keys \
                          >>.ssh/authorized_keys
    
      This is handy so you can setup port forward for local
      services like.
      EOT    
    path: /usr/local/sbin/00-menu-of-scripts.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      mkdir -p $HOME/bin
      mkdir -p $HOME/sbin
      echo "Setting up SSH access to $(HOSTNAME)"
      ssh-keygen
      echo -n "Enter your GitHub username: "
      read -r GITHUB_USER
      if [ "${GITHUB_USER}" != "" ]; then
        curl -L -o - "https://github.com/${GITHUB_USER}.keys" >>.ssh/authorized_keys
      fi
      echo 'export PATH="$HOME/bin:$HOME/sbin:$PATH"' >>"${HOME}/.bashrc"
      IP_ADDRESS=$(hostname -I | cut -d\  -f 1)
      cat <<EOT

      You have added ${HOME}/bin to the PATH in your .bashrc
      file. To update the path without logging out and in
      do
      
              . .bashrc
      
      If you're running solr you can use SSH on the host system
      to port forward http:127.0.0.1:8983/solr with

             ssh -L 8983:127.0.0.1:8983 ubuntu@${IP_ADDRESS}

      EOT
    path: /usr/local/sbin/01-setup-scripts.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      python3 -m pip install idutils
      python3 -m pip install sqlfluff
      python3 -m pip install pylint
      python3 -m pip install progressbar2
      python3 -m pip install pybtex
      python3 -m pip install feedgen
      python3 -m pip install python-dateutil
      cat <<EOT >$HOME/bin/ws
      #!/bin/bash
      echo "Enter Ctlr-c to exit web server"
      if [ "$1" != "" ]; then
      	python3 -m http.server "$1"
      else
      	python3 -m http.server
      fi
      EOT
      chmod 775 $HOME/bin/ws
    path: /usr/local/sbin/02-add-python-packages.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      cd
      sudo snap install go --classic
      export GOPATH=$HOME
      mkdir -p "${HOME}/bin"
      mkdir -p "${HOME}/src/github.com/caltechlibrary"
      for REPO in irdmtools datatools dataset newt; do
        cd "${HOME}/src/github.com/caltechlibrary"
        git clone "https://github.com/caltechlibrary/${REPO}"
        cd "${REPO}"
        make
        make install
        cd "${HOME}/src/github.com/caltechlibrary"
      done
      cd 
    path: /usr/local/sbin/03-add-go-and-caltechlibrary-tools.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
    path: /usr/local/sbin/04-ghcup-install.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      PATH=$HOME/.ghcup/bin:$PATH
      export PATH
      cd
      git clone https://github.com/jgm/pandoc src/pandoc
      cd src/pandoc
      make
      cp -v $(find . -type f -name pandoc) "${HOME}/bin"
      cp -v $(find . -type f -name pandoc) "${HOME}/bin/pandoc-server"
    path: /usr/local/sbin/05-pandoc-build.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      cd
      git clone https://github.com/PostgREST/postgrest src/postgrest
      cd src/postgrest
      stack build --install-ghc --copy-bins --local-bin-path $HOME/bin
      cd
    path: /usr/local/sbin/06-postgrest-build.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      cd
      CPU="$(uname -m)"
      case "${CPU}" in
         x86_64|amd64)
           curl -L -O https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.11.1/opensearch-dashboards-2.11.1-linux-x64.deb
           curl -L -O https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.deb
           sudo dpkg -i opensearch-2.11.1-linux-x64.deb
           sudo dpkg -i opensearch-dashboards-2.11.1-linux-x64.deb
           ;;
         arm64|aarch64)
           curl -L -O https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.11.1/opensearch-dashboards-2.11.1-linux-arm64.deb
           curl -L -O https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-arm64.deb
           sudo dpkg -i opensearch-2.11.1-linux-arm64.deb
           sudo dpkg -i opensearch-dashboards-2.11.1-linux-arm64.deb
           ;;
         *)
         echo "Unsupport CPU ${CPU} see <https://opensearch.org/downloads.html>"
         ;;
      esac
      sudo adduser "$USER" opensearch
      sudo systemctl daemon-reload
      sudo systemctl enable opensearch
      sudo systemctl start opensearch
      sudo systemctl enable opensearch-dashboards
      sudo systemctl start opensearch-dashboards
      IP_ADDRESS=$(hostname -I | cut -d\  -f 1)
      cat <<EOT
        Openseach should be setup and can the REST API can be reached on port 9200. You
        can use SSH to port forward that port with the following command on the host system.

             ssh -L 9200:localhost:9200 ubuntu@${IP_ADDRESS}

      EOT
    path: /usr/local/sbin/07-add-opensearch.bash
    owner: root:root
    permissions: '0775'
  - content: |
      #!/bin/bash
      cd
      echo "Checking CPU type to see if Solr 9.5 is supported."
      CPU="$(uname -m)"
      case "${CPU}" in
         arm32v7)
           echo "${CPU}, this install script is untested"
           ;;
         arm64v8)
           echo "${CPU}, this install script is untested"
           ;;
         ppc64le)
           echo "${CPU}, this install script is untested"
           ;;
         s390x)
           echo "${CPU}, this install script is untested"
           ;;
         x86_64|amd64)
           echo "${CPU}, should work on Multipass VM"
           ;;
         arm64|aarch64)
           echo "${CPU}, should work on Multipass VM"
           ;;
         *)
           echo "Unsupport CPU ${CPU} see <https://solr.apache.org/downloads.html>"
           exit 1
           ;;
      esac
      # Download the binary distribution from solr.apache.org
      curl -L -O https://dlcdn.apache.org/solr/solr/9.5.0/solr-9.5.0.tgz
      # Extract the installer script
      tar vxzf solr-9.5.0.tgz solr-9.5.0/bin/install_solr_service.sh \
                              solr-9.5.0/examples \
                              solr-9.5.0/docs
      # Run the installer as "root"
      sudo bash solr-9.5.0/bin/install_solr_service.sh solr-9.4.1.tgz
      # Add ubuntu user to solr group.
      sudo adduser $USER solr
      ulimit -Hc 65536
      ulimit -Hu 65536
      IP_ADDRESS=$(hostname -I | cut -d\  -f 1)
      cat <<EOT

        Solr should be running on port 8983 by default. You are ready
        to start exploring Solr. :-)

        Examples can be found in solr-9.5.0/examples

        If you're running solr you can use SSH on the host system
        to port forward http:127.0.0.1:8983/solr with

             ssh -L 8983:127.0.0.1:8983 ubuntu@${IP_ADDRESS}

      EOT
      
    path: /usr/local/sbin/08-add-solr.bash
    owner: root:root
    permissions: '0775'
package_upgrade: true
# Install additional packages on first boot
#
# Default: none
#
# if packages are specified, then package_update will be set to true
#
# packages may be supplied as a single package name or as a list
# with the format [<package>, <version>] wherein the specific
# package version will be installed.
packages:
#  - ubuntu-desktop
  - aptitude
  - tmux
  - micro
  - vim
  - emacs
  - tcsh
  - build-essential
  - autotools-dev
  - autoconf
  - automake
  - git
  - curl
  - lynx
  - make
  - pandoc
  - pkg-config
  - postgresql-16
  - postgresql-16-postgis
  - postgresql-16-postgis-scripts
  - postgresql-16-mysql-fdw
  - postgresql-plpython3-16
  - postgresql-16-pllua
  - postgresql-16-plpgsql-check
  - postgresql-16-semver
  - pgloader
  - pgtop
  - libncurses-dev
  - libffi-dev
  - libgmp-dev
  - libncurses5
  - zlib1g-dev
  - shellcheck
  - python3
  - python3-pip
  - libffi-dev
  - libffi6
  - libgmp-dev
  - libgmp10
  - libncurses-dev
  - libncurses5
  - libtinfo5
  - httpie
  - tree
  - openjdk-11-jdk