Last used stamper

helm/blueapi/README.md

@@ -44,6 +44,7 @@ A Helm chart deploying a worker pod that runs Bluesky plans
 | serviceAccount.create | bool | `false` |  |
 | serviceAccount.name | string | `""` |  |
 | startupProbe | object | `{"failureThreshold":5,"httpGet":{"path":"/healthz","port":"http"},"periodSeconds":10}` | A more lenient livenessProbe to allow the service to start fully. This is automatically disabled when in debug mode. |
+| timeStampCron.enabled | bool | `true` |  |
 | tolerations | list | `[]` | May be required to run on specific nodes (e.g. the control machine) |
 | tracing | object | `{"fastapi":{"excludedURLs":"/healthz"},"otlp":{"enabled":false,"protocol":"http/protobuf","server":{"host":"http://opentelemetry-collector.tracing","port":4318}}}` | Exclude health probe requests from tracing by default to prevent spamming |
 | volumeMounts | list | `[{"mountPath":"/config","name":"worker-config","readOnly":true}]` | Additional volumeMounts on the output StatefulSet definition. Define how volumes are mounted to the container referenced by using the same name. |

helm/blueapi/files/scripts/time-stamper.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Get PVCs belonging to this blueapi release
+ALL_PVC=$(kubectl get pvc -n  $RELEASE_NAMESPACE  \
+  -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | \
+  grep "^$RELEASE_NAME-scratch-")
+# Get all PVCs currently mounted by running pods
+MOUNTED_PVCS=$(kubectl get pods -n  $RELEASE_NAMESPACE  \
+  -o=jsonpath='{.items[*].spec.volumes[*].persistentVolumeClaim.claimName}' | tr ' ' '\n' | sort -u)
+NOW=$(date +%s)
+#loop through all the pvcs annotating ones thare are mounted or lack a last-used stamp
+for pvc in $ALL_PVC; do
+  # Checks if Annotation for last-used is empty
+  ANNOTATION=$(kubectl get pvc "$pvc" -n  $RELEASE_NAMESPACE  -o=jsonpath='{.metadata.annotations.last-used}')
+  # -z checks if ANNOTATION is empty, if its empty or mounted to updates last-used else it ignores it
+  if [ -z "$ANNOTATION" ]; then 
+    kubectl annotate --overwrite pvc "$pvc" -n  $RELEASE_NAMESPACE  last-used="$NOW"
+  elif echo "$MOUNTED_PVCS" | grep -qx "$pvc"; then
+    kubectl annotate --overwrite pvc "$pvc" -n  $RELEASE_NAMESPACE  last-used="$NOW"
+  fi
+done

helm/blueapi/templates/configmap.yaml

@@ -35,27 +35,14 @@ data:
 ---    
 {{- end }}
 
+---
+{{- if .Values.timeStampCron.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name : {{include "blueapi.fullname" . }}-pvc-stamper-script
 data:
-  time-stamper.sh: |
-    #!/bin/sh
-    # Get PVCs belonging to this blueapi release
-    ALL_PVC=$(kubectl get pvc -n {{ .Release.Namespace }} \
-      -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | \
-      grep "^{{ .Release.Name }}-scratch-")
-    # Get all PVCs currently mounted by running pods
-    MOUNTED_PVCS=$(kubectl get pods -n {{ .Release.Namespace }} \
-      -o=jsonpath='{.items[*].spec.volumes[*].persistentVolumeClaim.claimName}' | tr ' ' '\n' | sort -u)
-    NOW=$(date +%s)
-    #loop through all the pvcs annotating ones thare are mounted or lack a last-used stamp
-    for pvc in $ALL_PVC; do
-      ANNOTATION=$(kubectl get pvc "$pvc" -n {{ .Release.Namespace }} -o=jsonpath='{.metadata.annotations.last-used}')
-      if [ -z "$ANNOTATION" ]; then
-        kubectl annotate --overwrite pvc "$pvc" -n {{ .Release.Namespace }} last-used="$NOW"
-      elif echo "$MOUNTED_PVCS" | grep -qx "$pvc"; then
-        kubectl annotate --overwrite pvc "$pvc" -n {{ .Release.Namespace }} last-used="$NOW"
-      fi
-    done
+  {{- $files := .Files }}
+  time-stamper.sh: |-
+{{ $files.Get "files/scripts/time-stamper.sh" | indent 4 }}
+{{- end }}

helm/blueapi/templates/cronjob.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.timeStampCron.enabled }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -43,7 +44,7 @@ spec:
   jobTemplate:
     spec:
       # amount of attempts of labeling a pvc
-      backoffLimit: 0
+      backoffLimit: 3
       # job stops after 60 secounds
       activeDeadlineSeconds: 60
       template:
@@ -63,10 +64,16 @@ spec:
 
           containers:
           - name: last-used-stamper
+            env:
+              - name: RELEASE_NAME
+                value: {{ .Release.Name }}
+              - name: RELEASE_NAMESPACE
+                value: {{ .Release.Namespace }}
             volumeMounts:
               - name: {{include "blueapi.fullname" . }}-pvc-stamper-script
                 mountPath: /scripts
             image: bitnami/kubectl:latest
             imagePullPolicy: IfNotPresent
             command: ["/scripts/time-stamper.sh"]
           restartPolicy: OnFailure
+{{- end }}

helm/blueapi/values.schema.json

@@ -292,6 +292,14 @@
                 }
             }
         },
+        "timeStampCron": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                }
+            }
+        },
         "tolerations": {
             "description": "May be required to run on specific nodes (e.g. the control machine)",
             "type": "array"

helm/blueapi/values.yaml

@@ -224,6 +224,9 @@ initContainer:
     # -- Size of persistent volume
     size: "1Gi"
 
+timeStampCron:
+  enabled: true
+
 debug:
   # -- If enabled, runs debugpy, allowing port-forwarding to expose port 5678 or attached vscode instance
   enabled: false

src/blueapi/cli/cli.py

@@ -83,8 +83,24 @@ def is_str_dict(val: Any) -> TypeGuard[TaskParameters]:
 @click.option(
     "-c", "--config", type=Path, help="Path to configuration YAML file", multiple=True
 )
+@click.option(
+    "-v",
+    "--verbose",
+    "log_level",
+    flag_value="DEBUG",
+    help="Include DEBUG level logging output",
+)
+@click.option(
+    "-q",
+    "--quiet",
+    "log_level",
+    flag_value="ERROR",
+    help="Reduce logging noise to only show errors",
+)
 @click.pass_context
-def main(ctx: click.Context, config: tuple[Path, ...]) -> None:
+def main(
+    ctx: click.Context, config: tuple[Path, ...], log_level: str | None = None
+) -> None:
     # if no command is supplied, run with the options passed
 
     # Set umask to DLS standard
@@ -96,6 +112,9 @@ def main(ctx: click.Context, config: tuple[Path, ...]) -> None:
     except FileNotFoundError as fnfe:
         raise ClickException(f"Config file not found: {fnfe.filename}") from fnfe
 
+    if log_level:
+        config_loader.use_values({"logging": {"level": log_level}})
+
     loaded_config: ApplicationConfig = config_loader.load()
 
     set_up_logging(loaded_config.logging)

tests/unit_tests/cli/test_cli.py

@@ -1384,3 +1384,14 @@ def test_config_schema(
 def test_task_parameter_type(value, result):
     t = ParametersType()
     assert t.convert(value, None, None) == result
+
+
+@pytest.mark.parametrize(
+    "flag,level",
+    [("--verbose", "DEBUG"), ("--quiet", "ERROR"), ("-v", "DEBUG"), ("-q", "ERROR")],
+)
+def test_log_level_override(flag: str, level: str, runner: CliRunner):
+    with patch("blueapi.log.logging") as mock_log:
+        runner.invoke(main, [flag])
+        mock_log.getLogger().setLevel.assert_called_once_with(level)
+        mock_log.StreamHandler().setLevel.assert_called_once_with(level)