[rqd] Add frame recovery logic for docker mode (#1614)

Whenever RQD restarts it loses track of all the frames launched by it that haven't finished. This change adds a new configurable option to backup frame states to a file, that is used to recover the frame cache state and try to re-bind to the running frames. This first version only works on docker mode --------- Signed-off-by: Diego Tavares <[email protected]>
AcademySoftwareFoundation · Jan 23, 2025 · 219444b · 219444b
1 parent 011ebc6
commit 219444b
Show file tree

Hide file tree

Showing 7 changed files with 435 additions and 73 deletions.
diff --git a/VERSION.in b/VERSION.in
@@ -1 +1 @@
-1.4
+1.5
diff --git a/proto/rqd.proto b/proto/rqd.proto
@@ -117,6 +117,7 @@ message RunFrame {
     string os = 25;
     int64 soft_memory_limit = 26;
     int64 hard_memory_limit = 27;
+    int32 pid = 28;
 }
 
 message RunFrameSeq {

diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py
@@ -162,6 +162,11 @@
 DOCKER_MOUNTS = []
 DOCKER_SHELL_PATH = "/bin/sh"
 
+# Backup running frames cache. Backup cache is turned off if this path is set to
+# None or ""
+BACKUP_CACHE_PATH = ""
+BACKUP_CACHE_TIME_TO_LIVE_SECONDS = 60
+
 try:
     if os.path.isfile(CONFIG_FILE):
         # Hostname can come from here: rqutil.getHostname()
@@ -243,6 +248,12 @@
         if config.has_section(__host_env_var_section):
             RQD_HOST_ENV_VARS = config.options(__host_env_var_section)
 
+        if config.has_option(__override_section, "BACKUP_CACHE_PATH"):
+            BACKUP_CACHE_PATH = config.get(__override_section, "BACKUP_CACHE_PATH")
+        if config.has_option(__override_section, "BACKUP_CACHE_TIME_TO_LIVE_SECONDS"):
+            BACKUP_CACHE_TIME_TO_LIVE_SECONDS = config.getint(
+                __override_section, "BACKUP_CACHE_TIME_TO_LIVE_SECONDS")
+
         __docker_mounts = "docker.mounts"
         __docker_config = "docker.config"
         __docker_images = "docker.images"