- feature: add deno docker runner (#62)

* - fix: add deno docker runner

* - fix: hardened deno permissions

* - improve: added test to check deno permissions

* - improve: docker status

* - refactor: reset execution storage and ephemeral code

* - fix: concurrency errors

* - feature: rework log system to mix all of them per context

* - fix: typos in execution time

* - fix: compile errors

* - fix: docker mount windows

* - fix: added a test for file persistency

* - fix: async log collection and execution timeout

* - refactor: execution context

* - fix: storage issues

* - ci: bump version 083

* - ci: run tests in host

* - fix: logs

* - fix: log message

.github/workflows/pr-check.yml

@@ -67,5 +67,6 @@ jobs:
         env:
           CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
           EMBEDDING_API_URL : ${{ vars.EMBEDDING_API_URL }}
+          CI_FORCE_DENO_IN_HOST: true
         run: |
           npx nx run-many -t test --parallel=false --skip-nx-cache

.gitignore

@@ -15,3 +15,4 @@ libs/shinkai-tools-runner/shinkai-tools-runner-resources
 apps/shinkai-tool-aave-load-requester/src/bundled-resources
 launch.json
 .DS_Store
+libs/shinkai-tools-runner/shinkai-tools-runner-execution-storage

apps/shinkai-tool-playwright-example/src/index.ts

@@ -1,5 +1,6 @@
 import * as playwright from 'npm:[email protected]';
 import chromePaths from 'npm:[email protected]';
+import process from 'node:process';
 
 type Configurations = {
   chromePath?: string;
@@ -13,7 +14,11 @@ export const run: Run<Configurations, Parameters, Result> = async (
   configurations,
   parameters,
 ): Promise<Result> => {
-  const chromePath = configurations?.chromePath || chromePaths.chrome;
+  const chromePath =
+    configurations?.chromePath ||
+    process.env.CHROME_BIN ||
+    chromePaths.chrome ||
+    chromePaths.chromium;
   console.log('executing chrome from', chromePath);
   const browser = await playwright['chromium'].launch({
     executablePath: chromePath,

libs/shinkai-tools-runner/Cargo.toml

@@ -30,6 +30,8 @@ tempfile = "3.13.0"
 log = "0.4.22"
 once_cell = { version = "1.20.2" }
 env_logger = "0.11.5"
+anyhow = { version = "1.0.93" }
+chrono = { version = "0.4.38" }
 
 [build-dependencies]
 copy_to_output = "2.2.0"

libs/shinkai-tools-runner/src/lib.test.rs

@@ -1,4 +1,5 @@
 use std::env;
+use std::time::Duration;
 
 use serde_json::json;
 
@@ -161,24 +162,28 @@ async fn max_execution_time() {
         .is_test(true)
         .try_init();
     let js_code = r#"
-        async function run(configurations, parameters) {
+        async function run() {
             let startedAt = Date.now();
+            const sleepMs = 100;
             while (true) {
                 const elapse = Date.now() - startedAt;
-                console.log(`while true every ${500}ms, elapse ${elapse} ms`);
+                console.log(`while true sleeping ${sleepMs}ms, elapse ${elapse} ms`);
                 await new Promise(async (resolve) => {
                     setTimeout(() => {
                         resolve();
-                    }, parameters.timeoutMs);
+                    }, sleepMs);
                 });
             }
-
             return { data: true };
         }
 "#;
     let tool = Tool::new(js_code.to_string(), serde_json::Value::Null, None);
     let run_result = tool
-        .run(None, serde_json::json!({ "timeoutMs": 3200 }), Some(3000))
+        .run(
+            None,
+            serde_json::json!({ "timeoutMs": 100 }),
+            Some(Duration::from_secs(2)),
+        )
         .await;
     assert!(run_result.is_err());
     assert!(run_result.err().unwrap().message().contains("timed out"));

libs/shinkai-tools-runner/src/tools/container_utils.rs

@@ -0,0 +1,54 @@
+use std::process::Command;
+
+#[derive(Debug, PartialEq)]
+pub enum DockerStatus {
+    NotInstalled,
+    NotRunning,
+    Running,
+}
+
+/// Checks if Docker is available on the system by attempting to run 'docker info' command.
+/// This function verifies both that Docker is installed and that the Docker daemon is running.
+///
+/// # Details
+///
+/// The function executes `docker info` which requires:
+/// - Docker CLI to be installed and in PATH
+/// - Docker daemon to be running and accessible
+/// - Current user to have permissions to access Docker
+///
+/// # Returns
+///
+/// * `true` - If Docker is fully operational (installed, running and accessible)
+/// * `false` - If Docker is not available for any reason:
+///   - Docker is not installed
+///   - Docker daemon is not running
+///   - User lacks permissions
+///   - Other Docker configuration issues
+///
+/// # Example
+///
+/// ```
+/// use shinkai_tools_runner::tools::container_utils;
+///
+/// let docker_available = container_utils::is_docker_available();
+/// if docker_available {
+///     println!("docker is available and ready to use");
+/// } else {
+///     println!("docker is not available - check installation and permissions");
+/// }
+/// ```
+pub fn is_docker_available() -> DockerStatus {
+    let docker_check = Command::new("docker").arg("info").output();
+
+    match docker_check {
+        Ok(output) => {
+            if output.status.success() {
+                DockerStatus::Running
+            } else {
+                DockerStatus::NotRunning
+            }
+        }
+        Err(_) => DockerStatus::NotInstalled,
+    }
+}

libs/shinkai-tools-runner/src/tools/deno_execution_storage.rs

@@ -0,0 +1,153 @@
+use std::{
+    io::Write,
+    path::{self, Path, PathBuf},
+};
+
+use nanoid::nanoid;
+
+use super::execution_context::ExecutionContext;
+
+#[derive(Default, Clone)]
+pub struct DenoExecutionStorage {
+    pub context: ExecutionContext,
+    pub code_id: String,
+    pub root: PathBuf,
+    pub root_code: PathBuf,
+    pub code: PathBuf,
+    pub code_entrypoint: PathBuf,
+    pub deno_cache: PathBuf,
+    pub logs: PathBuf,
+    pub log_file: PathBuf,
+    pub home: PathBuf,
+}
+
+impl DenoExecutionStorage {
+    pub fn new(context: ExecutionContext) -> Self {
+        let code_id = format!("{}-{}", context.code_id, nanoid!());
+        let root =
+            path::absolute(context.storage.join(context.context_id.clone()).clone()).unwrap();
+        let root_code = path::absolute(root.join("code")).unwrap();
+        let code = path::absolute(root_code.join(code_id.clone())).unwrap();
+        let logs = path::absolute(root.join("logs")).unwrap();
+        let log_file = path::absolute(logs.join(format!(
+            "log_{}_{}.log",
+            context.context_id, context.execution_id,
+        )))
+        .unwrap();
+        let deno_cache = path::absolute(root.join("deno-cache")).unwrap();
+        Self {
+            context,
+            code_id: code_id.clone(),
+            root: root.clone(),
+            root_code,
+            code: code.clone(),
+            code_entrypoint: code.join("index.ts"),
+            deno_cache,
+            logs: logs.clone(),
+            log_file,
+            home: root.join("home"),
+        }
+    }
+
+    pub fn init(&self, code: &str, pristine_cache: Option<bool>) -> anyhow::Result<()> {
+        for dir in [
+            &self.root,
+            &self.root_code,
+            &self.code,
+            &self.deno_cache,
+            &self.logs,
+            &self.home,
+        ] {
+            log::info!("creating directory: {}", dir.display());
+            std::fs::create_dir_all(dir).map_err(|e| {
+                log::error!("failed to create directory {}: {}", dir.display(), e);
+                e
+            })?;
+        }
+
+        log::info!("creating code file: {}", self.code_entrypoint.display());
+        std::fs::write(&self.code_entrypoint, code).map_err(|e| {
+            log::error!("failed to write code to index.ts: {}", e);
+            e
+        })?;
+
+        log::info!(
+            "creating log file if not exists: {}",
+            self.log_file.display()
+        );
+        if !self.log_file.exists() {
+            std::fs::write(&self.log_file, "").map_err(|e| {
+                log::error!("failed to create log file: {}", e);
+                e
+            })?;
+        }
+
+        if pristine_cache.unwrap_or(false) {
+            std::fs::remove_dir_all(&self.deno_cache)?;
+            std::fs::create_dir(&self.deno_cache)?;
+            log::info!(
+                "cleared deno cache directory: {}",
+                self.deno_cache.display()
+            );
+        }
+
+        Ok(())
+    }
+
+    pub fn get_relative_code_entrypoint(&self) -> anyhow::Result<String> {
+        self.code_entrypoint
+            .strip_prefix(&self.root)
+            .map(|p| p.to_string_lossy().to_string())
+            .map_err(|e| {
+                log::error!("failed to get relative path: {}", e);
+                anyhow::anyhow!("failed to get relative path: {}", e)
+            })
+    }
+
+    pub fn get_relative_deno_cache(&self) -> anyhow::Result<String> {
+        self.deno_cache
+            .strip_prefix(&self.root)
+            .map(|p| p.to_string_lossy().to_string())
+            .map_err(|e| {
+                log::error!("failed to get relative path: {}", e);
+                anyhow::anyhow!("failed to get relative path: {}", e)
+            })
+    }
+
+    pub fn append_log(&self, log: &str) -> anyhow::Result<()> {
+        let timestamp = chrono::Local::now().format("%Y%m%d_%H%M%S");
+        let log_line = format!(
+            "{},{},{},{},{}\n",
+            timestamp, self.context.context_id, self.context.execution_id, self.code_id, log,
+        );
+        let mut file = std::fs::OpenOptions::new()
+            .append(true)
+            .create(true) // Create the file if it doesn't exist
+            .open(self.log_file.clone())
+            .map_err(|e| {
+                log::error!("failed to open log file: {}", e);
+                e
+            })?;
+        file.write_all(log_line.as_bytes())?;
+        Ok(())
+    }
+}
+
+// We do best effort to remove ephemereal folders
+impl Drop for DenoExecutionStorage {
+    fn drop(&mut self) {
+        if let Err(e) = std::fs::remove_dir_all(&self.code) {
+            log::warn!(
+                "failed to remove code directory {}: {}",
+                self.code.display(),
+                e
+            );
+        } else {
+            log::info!("removed code directory: {}", self.code.display());
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "deno_execution_storage.test.rs"]
+mod tests;