Hyperion supervisor implementation

docs/developer/hyperion/hyperion-blueapi.rst

@@ -23,7 +23,7 @@ Hyperion on BlueAPI consists of two components:
 Deployment
 ----------
 
-``hyperion-blueapi`` is automatically available in a standard Hyperion deployment.
+``hyperion-blueapi`` and ``hyperion-supervisor`` are automatically available in a standard Hyperion deployment.
 
 Launching
 ---------
@@ -33,3 +33,10 @@ Launching
 ::
 
     ./run_hyperion.sh --beamline=i03 --dev --blueapi
+
+
+``hyperion-supervisor`` can be launched using the ``run_hyperion.sh`` script, using the ``--supervisor`` option:
+
+::
+
+    ./run_hyperion.sh --beamline=i03 --dev --supervisor

run_hyperion.sh

@@ -9,6 +9,11 @@ MODE=gda
 
 CONFIG_DIR=`dirname $0`/src/mx_bluesky/hyperion
 BLUEAPI_CONFIG=$CONFIG_DIR/blueapi_config.yaml
+SUPERVISOR_CONFIG=$CONFIG_DIR/supervisor/supervisor_config.yaml
+CLIENT_CONFIG=$CONFIG_DIR/supervisor/client_config.yaml
+DO_CALLBACKS=1
+HEALTHCHECK_PORT=5005
+CALLBACK_WATCHDOG_PORT=5005
 
 for option in "$@"; do
     case $option in
@@ -26,12 +31,19 @@ for option in "$@"; do
         --dev)
             IN_DEV=true
             BLUEAPI_CONFIG=$CONFIG_DIR/blueapi_dev_config.yaml
+            SUPERVISOR_CONFIG=$CONFIG_DIR/supervisor/supervisor_dev_config.yaml
             ;;
         --udc)
             MODE=udc
             ;;
         --blueapi)
             MODE=blueapi
+            CALLBACK_WATCHDOG_PORT=5006
+            ;;
+        --supervisor)
+            MODE=supervisor
+            DO_CALLBACKS=0
+            HEALTHCHECK_PORT=5006
             ;;
         --help|--info|--h)
             source .venv/bin/activate
@@ -48,6 +60,8 @@ Options:
   --dev                   Enable dev mode to run from a local workspace on a development machine.
   --udc                   Start hyperion in UDC mode instead of taking commands from GDA
   --blueapi               Start hyperion in blueapi mode instead of taking commands from GDA
+  --supervisor            Start hyperion in supervisor mode, taking commands from Agamemnon and feeding them to
+                          an instance running in blueapi mode.
   --help                  This help
 
 By default this script will start an Hyperion server unless the --no-start flag is specified.
@@ -62,11 +76,18 @@ END
 done
 
 kill_active_apps () {
-    echo "Killing active instances of hyperion and hyperion-callbacks..."
-    pkill -e -f "python.*hyperion"
-    pkill -e -f "SCREEN.*hyperion"
-    blueapi controller stop 2>/dev/null
-    echo "done."
+    if [ $MODE = "supervisor" ]; then
+      # supervisor mode kills only supervisor
+      echo "Killing active instances of hyperion supervisor..."
+      pkill -e -f "mx-bluesky/.venv/bin/python .*--mode supervisor"
+    else
+      echo "Killing active instances of hyperion-blueapi"
+      pkill -e -f "python .*mx-bluesky/.venv/bin/blueapi .*serve"
+      echo "Killing vanilla hyperion instances"
+      pkill -e -f "mx-bluesky/.venv/bin/python .*--mode (gda|udc)"
+      echo "Killing hyperion-callbacks"
+      pkill -e -f "mx-bluesky/.venv/bin/python .*hyperion-callbacks"
+    fi
 }
 
 check_user () {
@@ -99,7 +120,7 @@ if [[ $START == 1 ]]; then
         check_user
         ISPYB_CONFIG_PATH="/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-${BEAMLINE}.cfg"
     else
-        ISPYB_CONFIG_PATH="$RELATIVE_SCRIPT_DIR/tests/test_data/ispyb_test_credentials.cfg"
+        ISPYB_CONFIG_PATH="$RELATIVE_SCRIPT_DIR/tests/test_data/ispyb-test-credentials.cfg"
         ZOCALO_CONFIG="$RELATIVE_SCRIPT_DIR/tests/test_data/zocalo-test-configuration.yaml"
         export ZOCALO_CONFIG
     fi
@@ -121,17 +142,24 @@ if [[ $START == 1 ]]; then
     fi
     echo "$(date) Logging to $LOG_DIR"
     export LOG_DIR
-    mkdir -p $LOG_DIR
-    start_log_path=$LOG_DIR/start_log.log
+    mkdir -p "$LOG_DIR"
+    if [ $MODE = "supervisor" ]; then
+      start_log_path=$LOG_DIR/supervisor_start_log.log
+    else
+      start_log_path=$LOG_DIR/start_log.log
+    fi
     callback_start_log_path=$LOG_DIR/callback_start_log.log
 
     source .venv/bin/activate
 
     declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" )
     declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" )
 
-    h_commands="--mode $MODE"
-    cb_commands=()
+    h_commands="--mode $MODE "
+    cb_commands="--watchdog-port $CALLBACK_WATCHDOG_PORT "
+    if [ $MODE = "supervisor" ]; then
+      h_commands+="--client-config ${CLIENT_CONFIG} --supervisor-config ${SUPERVISOR_CONFIG} "
+    fi
     for i in "${!h_and_cb_args[@]}"
     do
         if [ "${h_and_cb_args[$i]}" != false ]; then 
@@ -146,18 +174,20 @@ if [[ $START == 1 ]]; then
       blueapi --config $BLUEAPI_CONFIG serve > $start_log_path 2>&1 &
       HEALTHCHECK_ENDPOINT="healthz"
     else
-      echo "Starting hyperion with hyperion $h_commands, start_log is $start_log_path"
+      echo "Starting hyperion in mode $MODE with hyperion $h_commands, start_log is $start_log_path"
       hyperion `echo $h_commands;`>$start_log_path  2>&1 &
       HEALTHCHECK_ENDPOINT="status"
     fi
-    echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path"
-    hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 &
+    if [[ $DO_CALLBACKS == 1 ]]; then
+      echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path"
+      hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 &
+    fi
     echo "$(date) Waiting for Hyperion to start"
 
     for i in {1..30}
     do
         echo "$(date)"
-        curl --head -X GET http://localhost:5005/$HEALTHCHECK_ENDPOINT >/dev/null
+        curl --head -X GET http://localhost:$HEALTHCHECK_PORT/$HEALTHCHECK_ENDPOINT >/dev/null
         ret_value=$?
         if [ $ret_value -ne 0 ]; then
             sleep 1

src/mx_bluesky/hyperion/__main__.py

@@ -2,9 +2,11 @@
 import signal
 import threading
 from dataclasses import asdict
+from pathlib import Path
 from sys import argv
 from traceback import format_exception
 
+from blueapi.config import ApplicationConfig, ConfigLoader
 from blueapi.core import BlueskyContext
 from flask import Flask, request
 from flask_restful import Api, Resource
@@ -43,6 +45,7 @@
     StatusAndMessage,
     make_error_status_and_message,
 )
+from mx_bluesky.hyperion.supervisor import SupervisorRunner
 from mx_bluesky.hyperion.utils.context import setup_context
 
 
@@ -153,7 +156,11 @@ def create_app(runner: GDARunner, test_config=None) -> Flask:
 def initialise_globals(args: HyperionArgs):
     """Do all early main low-level application initialisation."""
     do_default_logging_setup(
-        CONST.LOG_FILE_NAME, CONST.GRAYLOG_PORT, dev_mode=args.dev_mode
+        CONST.SUPERVISOR_LOG_FILE_NAME
+        if args.mode == HyperionMode.SUPERVISOR
+        else CONST.LOG_FILE_NAME,
+        CONST.GRAYLOG_PORT,
+        dev_mode=args.dev_mode,
     )
     LOGGER.info(f"Hyperion launched with args:{argv}")
     alerting.set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
@@ -184,13 +191,32 @@ def main():
         case HyperionMode.UDC:
             context = setup_context(dev_mode=args.dev_mode)
             plan_runner = InProcessRunner(context, args.dev_mode)
-            create_server_for_udc(plan_runner)
+            create_server_for_udc(plan_runner, HyperionConstants.HYPERION_PORT)
             _register_sigterm_handler(plan_runner)
             run_forever(plan_runner)
         case HyperionMode.SUPERVISOR:
-            raise RuntimeError(
-                "Supervisor mode not supported yet see https://github.com/DiamondLightSource/mx-bluesky/issues/1365"
-            )
+            if not args.client_config:
+                raise RuntimeError(
+                    "BlueAPI client configuration file must be specified in supervisor mode."
+                )
+            if not args.supervisor_config:
+                raise RuntimeError(
+                    "BlueAPI supervisor configuration file must be specified in supervisor mode."
+                )
+
+            client_config = _load_config_from_yaml(Path(args.client_config))
+            supervisor_config = _load_config_from_yaml(Path(args.supervisor_config))
+            context = BlueskyContext(configuration=supervisor_config)
+            plan_runner = SupervisorRunner(context, client_config, args.dev_mode)
+            create_server_for_udc(plan_runner, HyperionConstants.SUPERVISOR_PORT)
+            _register_sigterm_handler(plan_runner)
+            run_forever(plan_runner)
+
+
+def _load_config_from_yaml(config_path: Path):
+    loader = ConfigLoader(ApplicationConfig)
+    loader.use_values_from_yaml(config_path)
+    return loader.load()
 
 
 def _register_sigterm_handler(runner: PlanRunner):

src/mx_bluesky/hyperion/blueapi_config.yaml

@@ -8,10 +8,9 @@ env:
     broadcast_status_events: false
 api:
   url: http://localhost:5005
-  cors:
-    allow_credentials: True
-    origins: 
-      - "*"
+stomp:
+  enabled: true
+  url: tcp://localhost:61613
 logging:
   graylog:
     url: "tcp://graylog-log-target.diamond.ac.uk:12232"

src/mx_bluesky/hyperion/blueapi_dev_config.yaml

@@ -9,9 +9,8 @@ env:
     broadcast_status_events: false
 api:
   url: http://localhost:5005
-  cors:
-    allow_credentials: True
-    origins: 
-      - "*"
+stomp:
+  enabled: true
+  url: tcp://localhost:61613
 logging:
   level: DEBUG

src/mx_bluesky/hyperion/blueapi_plans/__init__.py

@@ -73,13 +73,13 @@ def robot_unload(
 
 def clean_up_udc(
     visit: str,
-    cleanup_group="cleanup",
+    cleanup_group: str = "cleanup",
     robot: BartRobot = inject("robot"),
     smargon: Smargon = inject("smargon"),
     aperture_scatterguard: ApertureScatterguard = inject("aperture_scatterguard"),
     lower_gonio: XYZStage = inject("lower_gonio"),
     detector_motion: DetectorMotion = inject("detector_motion"),
-):
+) -> MsgGenerator:
     yield from bps.abs_set(
         detector_motion.shutter, ShutterState.CLOSED, group=cleanup_group
     )

src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py

@@ -52,8 +52,8 @@
 from mx_bluesky.hyperion.external_interaction.callbacks.snapshot_callback import (
     BeamDrawingCallback,
 )
-from mx_bluesky.hyperion.parameters.cli import parse_callback_dev_mode_arg
-from mx_bluesky.hyperion.parameters.constants import CONST, HyperionConstants
+from mx_bluesky.hyperion.parameters.cli import CallbackArgs, parse_callback_args
+from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.parameters.gridscan import (
     GridCommonWithHyperionDetectorParams,
     HyperionSpecifiedThreeDGridScan,
@@ -159,8 +159,8 @@ def wait_for_threads_forever(threads: Sequence[Thread]):
 class HyperionCallbackRunner:
     """Runs Nexus, ISPyB and Zocalo callbacks in their own process."""
 
-    def __init__(self, dev_mode) -> None:
-        setup_logging(dev_mode)
+    def __init__(self, callback_args: CallbackArgs) -> None:
+        setup_logging(callback_args.dev_mode)
         log_info("Hyperion callback process started.")
         set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
 
@@ -187,7 +187,12 @@ def start_dispatcher(callbacks: list[Callable]):
             name="0MQ Dispatcher",
         )
 
-        self.watchdog_thread = Thread(target=run_watchdog, daemon=True, name="Watchdog")
+        self.watchdog_thread = Thread(
+            target=run_watchdog,
+            daemon=True,
+            name="Watchdog",
+            args=[callback_args.watchdog_port],
+        )
         log_info("Created 0MQ proxy and local RemoteDispatcher.")
 
     def start(self):
@@ -201,12 +206,12 @@ def start(self):
         )
 
 
-def run_watchdog():
+def run_watchdog(watchdog_port: int):
     log_info("Hyperion watchdog keepalive running")
     while True:
         try:
             with request.urlopen(
-                f"http://localhost:{HyperionConstants.HYPERION_PORT}/callbackPing",
+                f"http://localhost:{watchdog_port}/callbackPing",
                 timeout=PING_TIMEOUT_S,
             ) as response:
                 if response.status != 200:
@@ -219,9 +224,10 @@ def run_watchdog():
 
 
 def main(dev_mode=False) -> None:
-    dev_mode = dev_mode or parse_callback_dev_mode_arg()
+    callback_args = parse_callback_args()
+    callback_args.dev_mode = dev_mode or callback_args.dev_mode
     print(f"In dev mode: {dev_mode}")
-    runner = HyperionCallbackRunner(dev_mode)
+    runner = HyperionCallbackRunner(callback_args)
     runner.start()
 
 

src/mx_bluesky/hyperion/parameters/cli.py

@@ -4,6 +4,7 @@
 from pydantic.dataclasses import dataclass
 
 from mx_bluesky._version import version
+from mx_bluesky.hyperion.parameters.constants import HyperionConstants
 
 
 class HyperionMode(StrEnum):
@@ -16,6 +17,14 @@ class HyperionMode(StrEnum):
 class HyperionArgs:
     mode: HyperionMode
     dev_mode: bool = False
+    client_config: str | None = None
+    supervisor_config: str | None = None
+
+
+@dataclass
+class CallbackArgs:
+    dev_mode: bool = False
+    watchdog_port: int = HyperionConstants.HYPERION_PORT
 
 
 def _add_callback_relevant_args(parser: argparse.ArgumentParser) -> None:
@@ -27,12 +36,17 @@ def _add_callback_relevant_args(parser: argparse.ArgumentParser) -> None:
     )
 
 
-def parse_callback_dev_mode_arg() -> bool:
+def parse_callback_args() -> CallbackArgs:
     """Returns the bool representing the 'dev_mode' argument."""
     parser = argparse.ArgumentParser()
     _add_callback_relevant_args(parser)
+    parser.add_argument(
+        "--watchdog-port",
+        type=int,
+        help="Liveness port for callbacks to ping regularly",
+    )
     args = parser.parse_args()
-    return args.dev
+    return CallbackArgs(dev_mode=args.dev, watchdog_port=args.watchdog_port)
 
 
 def parse_cli_args() -> HyperionArgs:
@@ -54,5 +68,17 @@ def parse_cli_args() -> HyperionArgs:
         type=HyperionMode,
         choices=HyperionMode.__members__.values(),
     )
+    parser.add_argument(
+        "--client-config", help="Specify the blueapi client configuration file."
+    )
+    parser.add_argument(
+        "--supervisor-config",
+        help="Specify the supervisor bluesky context configuration file.",
+    )
     args = parser.parse_args()
-    return HyperionArgs(dev_mode=args.dev or False, mode=args.mode)
+    return HyperionArgs(
+        dev_mode=args.dev or False,
+        mode=args.mode,
+        supervisor_config=args.supervisor_config,
+        client_config=args.client_config,
+    )