diff --git a/.dockerignore b/.dockerignore index ba7a552e17..61ead4d8b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,8 +5,10 @@ .ruff_cache **/__pycache__/ +# _version.py updated when image is built +**/_version.py + # virtualenv stuff - this gets built by the docker script .venv activate - tmp diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index e8172726dc..596f92521d 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -48,10 +48,10 @@ jobs: images: | ghcr.io/${{ github.repository }} labels: | - org.opencontainers.image.title=mx-bluesky + org.opencontainers.image.title=hyperion org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection annotations: | - org.opencontainers.image.title=mx-bluesky + org.opencontainers.image.title=hyperion org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection tags: | type=ref,event=tag diff --git a/Dockerfile.hyperion b/Dockerfile.hyperion index 3f84127d9d..2b2c4f4129 100644 --- a/Dockerfile.hyperion +++ b/Dockerfile.hyperion @@ -1,46 +1,59 @@ -FROM python:3.11 AS build +# This is the Dockerfile for building the image for hyperion-supervisor and hyperion-callbacks + +# The image will contain the directory structure: +# /app/mx-bluesky/.venv +# /app/mx-bluesky/.git +# /app/dodal/.git +# /app/mx-bluesky/scripts +# but we don't include the source folder structure for mx-bluesky or dodal since these will be +# deployed in a separate volume by the initContainer for the scratch folder + +FROM python:3.12 AS base-image ARG DEBIAN_FRONTEND=noninteractive +ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY # Need: # libgl, libglib2.0 for python cv2 dependency -RUN apt update -y && \ - apt upgrade -y && \ - apt install -y libgl1 libglib2.0-0 +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y libgl1 libglib2.0-0 + +# Install matching uv binaries for our python distro +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -RUN pip install --no-cache-dir --no-compile setuptools_scm +# Create the hyperion user and group which will be needed in the deployment securityContext +RUN groupadd -r -g 1000 hyperion && \ + useradd -r -u 1000 -g hyperion -d /app hyperion + +ADD --chown=hyperion:hyperion .git /app/mx-bluesky/.git +RUN mkdir -p /app/scripts && \ + chown hyperion:hyperion /app/scripts + +############################################################################# +FROM base-image as uv-staging # Copy the pyproject.toml and install dependencies for better caching when developing # & rerunning deployment scripts -COPY pyproject.toml /app/hyperion/ -WORKDIR "/app/hyperion" -RUN mkdir -p src/mx_bluesky - -# This enables us to cache the pip install without needing _version.py -# see https://setuptools-scm.readthedocs.io/en/latest/usage/ -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=1.0.0 pip install \ - --no-cache-dir --no-compile -e . - -# Check out and install dodal locally with no dependencies as this may be a different version to what -# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the -# running container -RUN mkdir ../dodal && \ -git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \ -pip install --no-cache-dir --no-compile --no-deps -e ../dodal - -# -# Everything above this line should be in the image cache unless pyproject.toml changes -# -ADD .git /app/hyperion/.git -# Restore the repository at the current commit instead of copying, to exclude uncommitted changes -# This is so that if you build a developer image from this dockerfile then _version.py will not -# append the dirty workdir hash, which causes complications during deployments that mount from a clean folder. -RUN git restore . - -# Regenerate _version.py with the correct version - this should run quickly since we already have our dependencies -RUN rm src/mx_bluesky/_version.py -RUN pip install --no-cache-dir --no-compile --no-deps -e . - -ENTRYPOINT ["/app/hyperion/utility_scripts/docker/entrypoint.sh"] - -EXPOSE 5005 +WORKDIR "/app/mx-bluesky" +COPY pyproject.toml uv.lock ./ +RUN mkdir -p src/mx_bluesky && \ + git config --global --add safe.directory /app/mx-bluesky && \ + git restore . && \ + uv sync --no-cache --locked + +############################################################################# +FROM base-image as build + +WORKDIR "/app/mx-bluesky" +RUN mkdir /scratch && \ + chown hyperion:hyperion /scratch + +USER hyperion:hyperion + +RUN git clone --bare https://github.com/DiamondLightSource/dodal.git /app/dodal/.git + +COPY --chown=hyperion:hyperion --from=uv-staging /app/mx-bluesky/.venv /app/mx-bluesky/.venv +COPY --chown=hyperion:hyperion utility_scripts/docker/initialise_scratch.sh /app/scripts/ + +EXPOSE 5006 diff --git a/docs/developer/general/explanations/containerised_mx_bluesky.rst b/docs/developer/general/explanations/containerised_mx_bluesky.rst index 51bcf3d8b3..850b260b16 100644 --- a/docs/developer/general/explanations/containerised_mx_bluesky.rst +++ b/docs/developer/general/explanations/containerised_mx_bluesky.rst @@ -3,12 +3,32 @@ Containerised mx-bluesky There are currently two images associated with this repository which are pushed on release: hyperion, and mx-bluesky-blueapi. -The Hyperion image exists because Hyperion was developed before BlueAPI was production-ready, and so doesn't use BlueAPI to schedule plans. This image is only really relevant for i03, and currently isn't used in production anywhere +The Hyperion image is the image that provides ``hyperion-supervisor`` and ``hyperion-callbacks``. These are launched as applications in their own right; ``hyperion-supervisor`` makes use of BlueAPI client but does not expose blueapi plans. ``hyperion-callbacks`` does not use BlueAPI. Which application is launched depends on whether ``hyperion`` or ``hyperion-callbacks`` is specified in the container launch command. -The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies. +The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies. This image can be used with BlueAPI's original helmchart, the only change required in the ``values.yaml`` is:: image: repository: ghcr.io/diamondlightsource/mx-bluesky-blueapi tag: "{desired_version}" + +``hyperion-blueapi`` is launched as a standard ``mx-bluesky-blueapi`` image with configuration to load the hyperion plan and beamline modules. + +Notes on the hyperion k8s deployment +------------------------------------ + +The hyperion Kubernetes deployment consists of a singled pod in a deployment which has 4 containers: + +* ``hyperion-init``, an initContainer which runs before all other containers start. +* ``hyperion-supervisor`` which launches the supervisor +* ``hyperion-callbacks`` which launches the external callbacks +* ``hyperion-scratch`` which is present to enable hotfixes to be applied + +The base ``hyperion`` container image contains only ``mx-bluesky`` and ``dodal`` bare git repos, plus the python virtual environment which provides all other library dependencies. When a new release of ``hyperion`` is first deployed to the cluster, it creates an empty Persistent Volume Claim (PVC). + +``hyperion-init`` runs on pod startup and is responsible for checking out ``dodal`` and ``mx-bluesky`` to the persistent volume if they do not already exist. + +``hyperion-supervisor`` and ``hyperion-callbacks`` then mount the persistent volume read-only under ``/scratch``, and then run ``hyperion`` from this. + +``hyperion-scratch`` is a container that has the persistent volume mounted read-write, its only purpose is to wait for VSCode to attach to it so that you can edit the writable PVC; since ordinary ephemeral containers cannot mount PVCs directly. In this manner, hot fixes to the code can be applied either by editing in VSCode, or via the console; when the pod is restarted the changes will be retained. diff --git a/helm/hyperion/Chart.yaml b/helm/hyperion/Chart.yaml index cea620e9b7..e786d03398 100644 --- a/helm/hyperion/Chart.yaml +++ b/helm/hyperion/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: hyperion -description: Hyperion server +description: Hyperion Supervisor and Callbacks type: application # version of the chart version: 0.0.1 diff --git a/helm/hyperion/templates/configmap.yaml b/helm/hyperion/templates/configmap.yaml new file mode 100644 index 0000000000..def19432d4 --- /dev/null +++ b/helm/hyperion/templates/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hyperion-supervisor-config +data: + supervisor_config.yaml: |- + # Configures the supervisor BlueAPI context to access the baton + env: + sources: + - kind: deviceManager + module: dodal.beamlines.{{ .Values.application.beamline }}_supervisor + mock: {{ .Values.application.dev }} + logging: + graylog: + url: {{ .Values.graylog.url }} + enabled: true + client_config.yaml: |- + # Configuration for the BlueAPI client running in the hyperion supervisor + api: + url: {{ .Values.blueapi.url }} + stomp: + enabled: true + url: {{ .Values.stomp.url }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hyperion-callbacks-config +data: + blueapi_callbacks.yml: |- + stomp: + enabled: true + url: {{ .Values.stomp.url }} diff --git a/helm/hyperion/templates/deployment.yaml b/helm/hyperion/templates/deployment.yaml index 41eb7ca251..b0ddf12391 100644 --- a/helm/hyperion/templates/deployment.yaml +++ b/helm/hyperion/templates/deployment.yaml @@ -6,6 +6,8 @@ spec: selector: matchLabels: app: hyperion + strategy: + type: Recreate replicas: 1 template: metadata: @@ -13,124 +15,153 @@ spec: app: hyperion spec: securityContext: - # gda2 - runAsUser: {{ .Values.application.runAsUser }} - runAsGroup: {{ .Values.application.runAsGroup }} - supplementalGroups: {{ .Values.application.supplementalGroups }} + fsGroup: {{ .Values.supervisor.runAsGroup }} + runAsGroup: {{ .Values.supervisor.runAsGroup }} + runAsUser: {{ .Values.supervisor.runAsUser }} volumes: - - name: dls-sw-bl - hostPath: - path: "/dls_sw/{{ .Values.application.beamline }}" - type: Directory - - name: dls-sw-dasc - hostPath: - path: "/dls_sw/dasc" - type: Directory - # Bind some source folders for easier debugging - - name: src - hostPath: - path: "{{ .Values.application.projectDir }}/src" - type: Directory - - name: tests - hostPath: - path: "{{ .Values.application.projectDir }}/tests" - type: Directory - - name: utility-scripts - hostPath: - path: "{{ .Values.application.projectDir }}/utility_scripts" - type: Directory - - name: dodal - hostPath: - path: "{{ .Values.dodal.projectDir | clean }}" - type: Directory - - name: logs - hostPath: - type: Directory - path: "{{ .Values.application.logDir }}" - - name: debuglogs - hostPath: - type: Directory - path: "{{ .Values.application.debugLogDir }}" - - name: data - hostPath: - type: Directory - path: "{{ .Values.application.dataDir }}" + - name: scratch + persistentVolumeClaim: + claimName: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }} + - name: supervisor-config + configMap: + name: hyperion-supervisor-config + - name: callbacks-config + configMap: + name: hyperion-callbacks-config + {{- toYaml .Values.volumes.logging | nindent 8 }} + {{- toYaml .Values.volumes.data | nindent 8 }} + - name: expeye-secrets + secret: + secretName: ispyb-hyperion-cfg - name: zocalo-config configMap: - name: "{{ .Release.Name }}-zocalo-config" + name: zocalo-config + defaultMode: 420 - name: zocalo-secrets - projected: - defaultMode: 0444 - sources: - {{- if .Values.zocalo.secrets }} - {{- range .Values.zocalo.secrets }} - - secret: - name: {{ .secretName }} - {{- end -}} {{/* range */}} - {{- end -}} {{/* if */}} - - name: ispyb-secrets secret: - type: secret - secretName: ispyb-hyperion-cfg + secretName: rmq-creds + initContainers: + - name: hyperion-init + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/app/scripts/initialise_scratch.sh'] + volumeMounts: + - name: scratch + mountPath: "/scratch" + env: + - name: DODAL_BRANCH + value: {{ .Values.initContainer.dodalBranch }} containers: - - name: hyperion - image: {{ .Values.application.imageRepository}}/mx-bluesky:{{ .Values.application.appVersion }} - resources: - limits: - cpu: "1" - memory: "1Gi" - ports: - - name: hyperion-api - containerPort: {{ .Values.application.containerPort }} - protocol: TCP - env: - - name: LOG_DIR - value: /var/log/bluesky - - name: DEBUG_LOG_DIR - value: /var/log/bluesky-debug - - name: BEAMLINE - value: "{{ .Values.application.beamline }}" - {{- if not .Values.application.dev }} - - name: ZOCALO_GO_USER - value: "gda2" - - name: ZOCALO_GO_HOSTNAME - value: "{{ .Values.application.beamline }}-control" - - name: ZOCALO_CONFIG - value: "/zocalo/config/configuration.yaml" - - name: ISPYB_CONFIG_PATH - value: "/ispyb/secrets/ispyb.cfg" - {{- end }} - readinessProbe: - exec: - command: [ "/app/hyperion/utility_scripts/docker/healthcheck.sh" ] - periodSeconds: 5 - volumeMounts: - - mountPath: "/dls_sw/{{ .Values.application.beamline }}" - name: dls-sw-bl - readOnly: true - mountPropagation: HostToContainer - - mountPath: "/dls_sw/dasc" - name: dls-sw-dasc - readOnly: true - mountPropagation: HostToContainer - - mountPath: "/app/hyperion/src" - name: src - - mountPath: "/app/hyperion/tests" - name: tests - - mountPath: "/app/hyperion/utility_scripts" - name: utility-scripts - - mountPath: "/app/dodal" - name: dodal - - mountPath: "/var/log/bluesky" - name: logs - - mountPath: "/var/log/bluesky-debug" - name: debuglogs - - mountPath: "/dls/{{ .Values.application.beamline }}/data" - name: data - - mountPath: "/zocalo/config" - name: zocalo-config - - mountPath: "/zocalo/secrets" - name: zocalo-secrets - - mountPath: "/ispyb/secrets" - name: ispyb-secrets - hostNetwork: true + - name: hyperion-supervisor + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/bin/bash', '-c'] + args: + - | + . /app/mx-bluesky/.venv/bin/activate + hyperion --mode supervisor \ + --client-config /etc/hyperion/client_config.yaml \ + --supervisor-config /etc/hyperion/supervisor_config.yaml + {{- if .Values.application.dev }} \ + --dev + sleep 600 + {{- end }} + resources: + limits: + cpu: "1" + memory: "1Gi" + ports: + - name: rest-api + containerPort: {{ .Values.supervisor.containerPort }} + protocol: TCP + env: + - name: LOG_DIR + value: /var/log/bluesky + - name: DEBUG_LOG_DIR + value: /var/log/bluesky/debug + - name: BEAMLINE + value: "{{ .Values.application.beamline }}" + - name: CONFIG_SERVER_URL + value: "{{ .Values.configServer.url }}" + - name: PYTHONPATH + value: "/scratch/mx-bluesky/src:/scratch/dodal/src" + readinessProbe: + httpGet: + path: "/status" + port: {{ .Values.supervisor.containerPort }} + initialDelaySeconds: 30 + periodSeconds: 5 + volumeMounts: + - mountPath: "/scratch" + name: scratch + readOnly: true + - mountPath: "/etc/hyperion" + name: supervisor-config + readOnly: true + - mountPath: "/var/log/bluesky" + name: logdir + - name: hyperion-callbacks + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/bin/bash', '-c'] + args: + - | + . /app/mx-bluesky/.venv/bin/activate + hyperion-callbacks \ + --watchdog-port {{ .Values.supervisor.containerPort }} \ + --stomp-config /etc/hyperion/blueapi_callbacks.yml + {{- if .Values.application.dev }} \ + --dev + sleep 600 + {{- end }} + securityContext: + fsGroup: {{ .Values.callbacks.runAsGroup }} + runAsGroup: {{ .Values.callbacks.runAsGroup }} + runAsUser: {{ .Values.callbacks.runAsUser }} + resources: + limits: + cpu: "1" + memory: "1Gi" + env: + - name: LOG_DIR + value: /var/log/bluesky + - name: DEBUG_LOG_DIR + value: /var/log/bluesky/debug + - name: BEAMLINE + value: "{{ .Values.application.beamline }}" + - name: CONFIG_SERVER_URL + value: "{{ .Values.configServer.url }}" + - name: PYTHONPATH + value: "/scratch/mx-bluesky/src:/scratch/dodal/src" + - name: ISPYB_CONFIG_PATH + value: "/etc/secrets/expeye/ispyb.cfg" + volumeMounts: + - mountPath: "/scratch" + name: scratch + readOnly: true + - mountPath: "/etc/hyperion" + name: callbacks-config + readOnly: true + - mountPath: "/etc/zocalo" + name: zocalo-config + readOnly: true + - mountPath: "/var/log/bluesky" + name: logdir + - mountPath: "/dls/{{ .Values.application.beamline }}/data" + name: data + - mountPath: "/etc/secrets/zocalo" + name: zocalo-secrets + - mountPath: "/etc/secrets/expeye" + name: expeye-secrets + - name: hyperion-scratch + # container provided in order for vscode to attach to a writable scratch + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + restartPolicy: Always + volumeMounts: + - mountPath: "/scratch" + name: scratch + resources: + limits: + cpu: "1" + memory: "1Gi" + command: ['/bin/bash', '-c'] + args: + - | + while true; do sleep 60; done diff --git a/helm/hyperion/templates/ingress.yaml b/helm/hyperion/templates/ingress.yaml deleted file mode 100644 index 446cf4a9e1..0000000000 --- a/helm/hyperion/templates/ingress.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if not .Values.application.dev }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: hyperion-ingress -spec: - ingressClassName: nginx - tls: - - hosts: - - {{ .Values.application.externalHostname }} - rules: - - host: {{ .Values.application.externalHostname }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hyperion-svc # this must match the name of the service you want to target - port: - number: {{ .Values.application.containerPort }} - {{- end }} - diff --git a/helm/hyperion/templates/service.yaml b/helm/hyperion/templates/service.yaml index ca11417892..222f70e195 100644 --- a/helm/hyperion/templates/service.yaml +++ b/helm/hyperion/templates/service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: hyperion-svc + name: hyperion-supervisor-svc spec: {{- if .Values.application.dev }} type: LoadBalancer @@ -9,9 +9,9 @@ spec: type: ClusterIP {{- end }} ports: - - name: hyperion-api - port: {{ .Values.application.servicePort }} + - name: hyperion-supervisor-rest + port: {{ .Values.supervisor.servicePort }} protocol: TCP - targetPort: {{ .Values.application.containerPort }} + targetPort: {{ .Values.supervisor.containerPort }} selector: - app: hyperion + app: hyperion-supervisor diff --git a/helm/hyperion/templates/volumes.yaml b/helm/hyperion/templates/volumes.yaml new file mode 100644 index 0000000000..5cd83135ef --- /dev/null +++ b/helm/hyperion/templates/volumes.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }} + annotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false + argocd.argoproj.io/compare-options: IgnoreExtraneous +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.scratch.volumeSize }} diff --git a/helm/hyperion/templates/zocalo-config.yaml b/helm/hyperion/templates/zocalo-config.yaml index 3fd1af8a36..0ce7498ad8 100644 --- a/helm/hyperion/templates/zocalo-config.yaml +++ b/helm/hyperion/templates/zocalo-config.yaml @@ -1,24 +1,22 @@ apiVersion: v1 kind: ConfigMap metadata: - name: "{{ .Release.Name }}-zocalo-config" + name: "zocalo-config" namespace: "{{ .Release.Namespace }}" labels: app.kubernetes.io/name: zocalo-config app.kubernetes.io/instance: "{{ .Release.Name }}-zocalo-config" app.kubernetes.io/version: "{{ .Chart.Version }}" - app.kubernetes.io/part-of: hyperion + app.kubernetes.io/part-of: hyperion data: configuration.yaml: |- version: 1 - graylog: plugin: graylog {{- .Values.zocalo.graylog | toYaml | nindent 6 }} - {{- if .Values.zocalo.secrets }} {{- range .Values.zocalo.secrets }} - {{ .name }}: "/zocalo/secrets/{{ .fileName }}" + {{ .name }}: "/etc/secrets/zocalo/{{ .fileName }}" {{- end -}} {{/* range */}} {{- end -}} {{/* if */}} diff --git a/helm/hyperion/values.yaml b/helm/hyperion/values.yaml index 0cd408b5c9..31d1aca02e 100644 --- a/helm/hyperion/values.yaml +++ b/helm/hyperion/values.yaml @@ -1,25 +1,45 @@ application: - containerPort: 5005 - servicePort: 80 imageRepository: ghcr.io/diamondlightsource - # i03-hyperion user and group - runAsUser: 36101 - runAsGroup: 36101 - supplementalGroups: [] + imageName: hyperion + imageVersion: SET_ON_INSTALL beamline: i03 dev: false - logDir: "/dls_sw/i03/logs/bluesky/hyperion-k8s" - debugLogDir: "/dls/tmp/i03/logs/bluesky-k8s" - dataDir: "/dls/i03/data" - # These should be overridden at install time - projectDir: SET_ON_INSTALL - appVersion: SET_ON_INSTALL - externalHostname: i03-hyperion.diamond.ac.uk -dodal: - projectDir: SET_ON_INSTALL -service: - type: NodePort - + logDir: "/var/logs/mx-bluesky" +supervisor: + servicePort: 80 + containerPort: 5006 + runAsUser: 1000 + runAsGroup: 1000 +callbacks: + runAsUser: 1000 + runAsGroup: 1000 +initContainer: + dodalBranch: "main" +scratch: + volumeSize: "2Gi" +configServer: + url: http://i03-daq-config-server.diamond.ac.uk +graylog: + url: "tcp://graylog-log-target.diamond.ac.uk:12232" +stomp: + url: tcp://localhost:61613 +blueapi: + url: http://my.blueapi.service:5005 +volumes: + logging: + - name: logdir + emptyDir: {} +# In ixx-services, replace with path to relevant log folder, e.g. +# hostPath: +# path: "/dls_sw/i03/logs/bluesky/hyperion-k8s" +# type: Directory + data: + - name: data + emptyDir: {} +# In ixx-services, replace with path to relevant data folder, e.g. +# hostPath: +# type: Directory +# path: "/dls/i03/data" zocalo: defaultTransport: "PikaTransport" graylog: @@ -30,6 +50,3 @@ zocalo: - name: rabbitmq secretName: rmq-creds fileName: rabbitmq-credentials.yml - - name: rabbitmq-api-reader - secretName: rmq-api-reader - fileName: rabbitmq-api-reader.yml diff --git a/helm/rabbitmq/Chart.yaml b/helm/rabbitmq/Chart.yaml new file mode 100644 index 0000000000..0ea3296092 --- /dev/null +++ b/helm/rabbitmq/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: mx-bluesky-rabbitmq-test +description: Rabbitmq server for use in testing environments + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +dependencies: + - name: rabbitmq + version: 2.3.0 + repository: https://groundhog2k.github.io/helm-charts/ diff --git a/helm/rabbitmq/README.md b/helm/rabbitmq/README.md new file mode 100644 index 0000000000..96edb237ab --- /dev/null +++ b/helm/rabbitmq/README.md @@ -0,0 +1 @@ +This helmchart is provided for development testing purposes only diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh index 6854d10697..1e96747fe4 100755 --- a/utility_scripts/build_docker_image.sh +++ b/utility_scripts/build_docker_image.sh @@ -1,8 +1,9 @@ #!/bin/bash -set -e -# builds the docker image +set -eo pipefail +# Script for building development docker images BUILD=1 -PUSH=1 +PUSH=0 +BUILD_UNCLEAN=0 PODMAN_FLAGS="" for option in "$@"; do case $option in @@ -10,22 +11,27 @@ for option in "$@"; do BUILD=0 shift ;; - --no-push) - PUSH=0 + --push) + PUSH=1 shift ;; --no-cache) PODMAN_FLAGS+=" --no-cache" shift ;; + --unclean) + BUILD_UNCLEAN=1 + shift + ;; --help|--info|--h) CMD=`basename $0` echo "$CMD [options]" - echo "Builds and/or pushes the docker container image to the repository" + echo "Builds a development docker image and optionally pushes the docker container image to the repository" echo " --help This help" echo " --no-build Do not build the image" - echo " --no-push Do not push the image" + echo " --push Push the image" echo " --no-cache Don't use the cache when building the image." + echo " --unclean Build with an unclean workspace" exit 0 ;; -*|--*) @@ -36,33 +42,33 @@ for option in "$@"; do done PROJECTDIR=`dirname $0`/.. -IMAGE=mx-bluesky +IMAGE=hyperion -if ! git diff --cached --quiet; then - echo "Cannot build image from unclean workspace" - exit 1 -fi +function extract_version() { + LATEST_VERSION=$(git tag | sed -E -n 's/^v?([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+)$/\1/p' | sort -V -r | head -1) + GIT_HASH=$(git rev-parse --short HEAD) + echo $LATEST_VERSION-g$GIT_HASH +} +if [[ $BUILD_UNCLEAN == 0 ]]; then + if ! git diff --cached --quiet; then + echo "Cannot build image from unclean workspace" + exit 1 + fi +fi if [[ $BUILD == 1 ]]; then echo "Building initial image" - LATEST_TAG=$IMAGE:latest + IMAGE_VERSION=$(extract_version) + MX_BLUESKY_VERSION=${IMAGE_VERSION/-/+} + LATEST_TAG=$IMAGE:dev TMPDIR=/tmp podman build \ $PODMAN_FLAGS \ + --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=$MX_BLUESKY_VERSION \ -f $PROJECTDIR/Dockerfile.hyperion \ --tag $LATEST_TAG \ + --tag $IMAGE:$IMAGE_VERSION \ $PROJECTDIR - # Now extract the version from the built image and then rebuild with the label - IMAGE_VERSION=$(podman run --rm --entrypoint=hyperion $LATEST_TAG -c "--version" | \ - sed -e 's/[^a-zA-Z0-9 ._-]/_/g') - TAG=$IMAGE:$IMAGE_VERSION - echo "Labelling image with version $IMAGE_VERSION, tagging with tags $TAG $LATEST_TAG" - TMPDIR=/tmp podman build \ - -f $PROJECTDIR/Dockerfile.hyperion \ - --tag $TAG \ - --tag $LATEST_TAG \ - --label "version=$IMAGE_VERSION" \ - $PROJECTDIR fi if [[ $PUSH == 1 ]]; then @@ -71,7 +77,7 @@ if [[ $PUSH == 1 ]]; then echo "Not logged in to ghcr.io" exit 1 fi - echo "Pushing to ghcr.io/$NAMESPACE/$IMAGE:latest ..." - podman push $IMAGE:latest docker://ghcr.io/$NAMESPACE/$IMAGE:latest - podman push $IMAGE:latest docker://ghcr.io/$NAMESPACE/$IMAGE:$IMAGE_VERSION + echo "Pushing to ghcr.io/$NAMESPACE/$IMAGE:dev ..." + podman push $IMAGE:dev docker://ghcr.io/$NAMESPACE/$IMAGE:dev + podman push $IMAGE:dev docker://ghcr.io/$NAMESPACE/$IMAGE:$IMAGE_VERSION fi diff --git a/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh b/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh index 59f70f8f2d..8a34955d99 100755 --- a/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh +++ b/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh @@ -1,7 +1,7 @@ #!/bin/bash # Installs helm package to kubernetes LOGIN=true - +LINT=false for option in "$@"; do case $option in -b=*|--beamline=*) @@ -12,28 +12,24 @@ for option in "$@"; do DEV=true shift ;; - --checkout-to-prod) - CHECKOUT=true - shift - ;; --repository=*) REPOSITORY="${option#*=}" shift ;; - --appVersion=*) - APP_VERSION="${option#*=}" + --imageVersion=*) + IMAGE_VERSION="${option#*=}" shift ;; --no-login) LOGIN=false shift ;; - --dry-run) - DRY_RUN=true + --lint) + LINT=true shift ;; - --bind-dir=*) - BIND_DIR="${option#*=}" + --dry-run) + DRY_RUN=true shift ;; --help|--info|--h) @@ -53,16 +49,15 @@ Arguments: Options: --help This help - --appVersion=version Version of the image to fetch from the repository otherwise it is deduced + --imageVersion=version Version of the image to fetch from the repository otherwise it is deduced from the setuptools_scm. Must be in the format x.y.z -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline - --checkout-to-prod Checkout source folders to the production folder using deploy_mx_bluesky.py --dev Install to a development kubernetes cluster (assumes project checked out under /home) (default cluster is argus in user namespace) --dry-run Do everything but don't do the final deploy to k8s --no-login Do not attempt to log in to kubernetes instead use the current namespace and cluster --repository=REPOSITORY Override the repository to fetch the image from - --bind-dir=DIR Override the directory to bind to, default calculated from app version + --lint Lint the helm chart EOM exit 0 ;; @@ -103,44 +98,6 @@ PROJECTDIR=$(readlink -e $(dirname $0)/../..) TOP_HELMCHART_DIR=${PROJECTDIR}/helm HELMCHART_DIR=${TOP_HELMCHART_DIR}/${APP_NAME} -if [[ -n $DEV ]]; then - if [[ -n $CHECKOUT ]]; then - echo "Cannot specify both --dev and --checkout-to-prod" - exit 1 - fi - CHECKED_OUT_VERSION=$(git describe --tag) -else - if [[ -z ${VIRTUAL_ENV#${PROJECTDIR}} ]]; then - echo "Virtual env not activated, activating" - . $PROJECTDIR/.venv/bin/activate - fi - - # First extract the version and location that will be deployed - DEPLOY_MX_BLUESKY="python $PROJECTDIR/utility_scripts/deploy/deploy_mx_bluesky.py" - MX_BLUESKY_BASE=$($DEPLOY_MX_BLUESKY --print-release-dir $BEAMLINE) - - if [[ -n $CHECKOUT ]]; then - if [[ -n $BIND_DIR ]]; then - echo "Cannot specify --bind-dir and --checkout-to-prod" - exit 1 - fi - echo "Running deploy_mx_bluesky.py to deploy to production folder..." - $DEPLOY_MX_BLUESKY --kubernetes $BEAMLINE - if [[ $? != 0 ]]; then - echo "Deployment failed, aborting." - exit 1 - fi - fi - - NEW_PROJECTDIR=$MX_BLUESKY_BASE/mx-bluesky - echo "Changing directory to $NEW_PROJECTDIR..." - cd $NEW_PROJECTDIR - PROJECTDIR=$NEW_PROJECTDIR - MX_BLUESKY_BASENAME=$(basename $MX_BLUESKY_BASE) - CHECKED_OUT_VERSION=${MX_BLUESKY_BASENAME#mx-bluesky_} -fi - - if [[ $LOGIN = true ]]; then if [[ -n $DEV ]]; then CLUSTER=argus @@ -151,73 +108,44 @@ if [[ $LOGIN = true ]]; then fi fi -ensure_version_py() { - # We require the _version.py to be created, this needs a minimal virtual environment - if [[ ! -d $PROJECTDIR/.venv ]]; then - echo "Creating _version.py" - echo "Virtual environment not found - creating" - module load python/3.11 && module load uv - uv venv $PROJECTDIR/.venv - source $PROJECTDIR/.venv/bin/activate - fi - uv pip install setuptools_scm -} - -app_version() { - . $PROJECTDIR/.venv/bin/activate - python -m setuptools_scm --force-write-version-files | sed -e 's/[^a-zA-Z0-9._-]/_/g' -} - if [[ -n $REPOSITORY ]]; then HELM_OPTIONS+="--set application.imageRepository=$REPOSITORY " fi -ensure_version_py -if [[ -z $APP_VERSION ]]; then - APP_VERSION=$(app_version) -fi - -echo "Checked out version that will be bind-mounted in $PROJECTDIR is $CHECKED_OUT_VERSION" echo "Container image version that will be pulled is $APP_VERSION" -if [[ $APP_VERSION != $CHECKED_OUT_VERSION ]]; then - echo "*****************************************************************" - echo "WARNING: Checked out version and container image versions differ!" - echo "*****************************************************************" -fi - +#application.runAsUser=$EUID,\ +#application.runAsGroup=$GID,\ +#application.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ +#application.externalHostname=test-$APP_NAME.diamond.ac.uk " if [[ -n $DEV ]]; then GID=`id -g` SUPPLEMENTAL_GIDS=37904 HELM_OPTIONS+="--set \ application.dev=true,\ -application.runAsUser=$EUID,\ -application.runAsGroup=$GID,\ -application.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ application.logDir=$PROJECTDIR/tmp,\ application.dataDir=$PROJECTDIR/tmp/data,\ -application.externalHostname=test-$APP_NAME.diamond.ac.uk " - mkdir -p $PROJECTDIR/tmp/data - DEPLOYMENT_DIR=$PROJECTDIR -else - DEPLOYMENT_DIR=/dls_sw/$BEAMLINE/software/bluesky/mx-bluesky_${APP_VERSION}/mx-bluesky -fi -if [[ -n $BIND_DIR ]]; then - DEPLOYMENT_DIR=$BIND_DIR +stomp.url=tcp://rabbitmq-test:61613 " fi -HELM_OPTIONS+="--set application.appVersion=v$APP_VERSION,\ -application.projectDir=$DEPLOYMENT_DIR,\ -dodal.projectDir=$DEPLOYMENT_DIR/../dodal " +HELM_OPTIONS+="--set application.imageVersion=$IMAGE_VERSION " module load helm +APP_VERSION=$IMAGE_VERSION + helm package $HELMCHART_DIR --app-version $APP_VERSION # Helm package generates a file suffixed with the chart version if [[ $LOGIN = true ]]; then module load $CLUSTER kubectl config set-context --current --namespace=$NAMESPACE fi -if [[ -z $DRY_RUN ]]; then +if [[ $DRY_RUN = true ]]; then + HELM_OPTIONS+="--dry-run=server --debug " +fi +if [[ $LINT = true ]]; then + helm template $RELEASE $HELMCHART_DIR $HELM_OPTIONS + helm lint $HELMCHART_DIR --strict --values $HELMCHART_DIR/values.yaml $HELM_OPTIONS +else helm upgrade --install $HELM_OPTIONS $RELEASE $APP_NAME-0.0.1.tgz fi diff --git a/utility_scripts/docker/entrypoint.sh b/utility_scripts/docker/entrypoint.sh deleted file mode 100755 index 9fba523722..0000000000 --- a/utility_scripts/docker/entrypoint.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Entry point for the production docker image that launches the external callbacks -# as well as the main server - -for option in "$@"; do - case $option in - --dev) - IN_DEV=true - ;; - --help|--info|--h) - echo "Arguments:" - echo " --dev start in development mode without external callbacks" - exit 0 - ;; - -*|--*) - echo "Unknown option ${option}. Use --help for info on option usage." - exit 1 - ;; - esac -done - -kill_active_apps () { - echo "Killing active instances of hyperion and hyperion-callbacks..." - pkill -e -f "python.*hyperion" - pkill -e -f "SCREEN.*hyperion" - echo "done." -} - -RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; ) -cd ${RELATIVE_SCRIPT_DIR} - -echo "$(date) Logging to $LOG_DIR" -mkdir -p $LOG_DIR -start_log_path=$LOG_DIR/start_log.log -callback_start_log_path=$LOG_DIR/callback_start_log.log - -#Add future arguments here -declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" ) -declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" ) - -h_commands=() -cb_commands=() -for i in "${!h_and_cb_args[@]}" -do - if [ "${h_and_cb_args[$i]}" != false ]; then - h_commands+="${h_and_cb_arg_strings[$i]} "; - cb_commands+="${h_and_cb_arg_strings[$i]} "; - fi; -done - -trap kill_active_apps TERM - -hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & - -echo "$(date) Starting Hyperion..." -hyperion `echo $h_commands;`>$start_log_path 2>&1 diff --git a/utility_scripts/docker/initialise_scratch.sh b/utility_scripts/docker/initialise_scratch.sh new file mode 100755 index 0000000000..9b0cc218b4 --- /dev/null +++ b/utility_scripts/docker/initialise_scratch.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This script is run in the initContainer before the main hyperion pod is launched to initialise the scratch area +# if not already present. + +SCRATCH_ROOT=/scratch +APP_ROOT=/app + +if [ ! -d $SCRATCH_ROOT/dodal/.git ]; then + git clone /app/dodal/.git $SCRATCH_ROOT/dodal + echo "Checking out dodal branch $DODAL_BRANCH" + git --git-dir=$SCRATCH_ROOT/dodal/.git checkout $DODAL_BRANCH +fi + +if [ ! -d $SCRATCH_ROOT/mx-bluesky/.git ]; then + CURRENT_BRANCH=$(git --git-dir=$APP_ROOT/mx-bluesky/.git rev-parse --abbrev-ref HEAD) + git clone /app/mx-bluesky/.git $SCRATCH_ROOT/mx-bluesky + ls -la $APP_ROOT/mx-bluesky/.git + echo "Checking out $CURRENT_BRANCH... as $UID" + ls -la $SCRATCH_ROOT/mx-bluesky/.git + git --git-dir=$SCRATCH_ROOT/mx-bluesky/.git checkout $CURRENT_BRANCH +fi + +# dodal and mx-bluesky are installed with uv pip install however since the writable layer for the .venv is ephemeral +# they still need to be added to PYTHONPATH in the container +cd $APP_ROOT/mx-bluesky +. ./.venv/bin/activate +uv pip install -e $SCRATCH_ROOT/mx-bluesky +uv pip install -e $SCRATCH_ROOT/dodal diff --git a/uv.lock b/uv.lock index 79e05c7d68..a54d860057 100644 --- a/uv.lock +++ b/uv.lock @@ -810,8 +810,8 @@ wheels = [ [[package]] name = "dls-dodal" -version = "2.4.1.dev7+g50b24b2c3" -source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#50b24b2c370e7214b97d25e03e4eaaf55517a7e1" } +version = "2.4.1.dev8+gbde3b691f" +source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#bde3b691faff4afb4f3fd7540b6bd6fcbead2b45" } dependencies = [ { name = "aiofiles", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },