Merge pull request #68 from threefoldtech/development_support_convers…

…ion_progress

support conversion progress

docker2fl/Cargo.toml

@@ -31,3 +31,4 @@ toml = "0.4.2"
 clap = { version = "4.2", features = ["derive"] }
 serde = { version = "1.0.159" , features = ["derive"] }
 tokio-async-drop = "0.1.0"
+walkdir = "2.5.0"

docker2fl/src/docker2fl.rs

@@ -4,6 +4,9 @@ use bollard::container::{
 };
 use bollard::image::{CreateImageOptions, RemoveImageOptions};
 use bollard::Docker;
+use std::sync::mpsc::Sender;
+use tempdir::TempDir;
+use walkdir::WalkDir;
 
 use anyhow::{Context, Result};
 use futures_util::stream::StreamExt;
@@ -18,8 +21,6 @@ use tokio_async_drop::tokio_async_drop;
 use rfs::fungi::Writer;
 use rfs::store::Store;
 
-use uuid::Uuid;
-
 struct DockerInfo {
     image_name: String,
     container_name: String,
@@ -43,46 +44,87 @@ impl Drop for DockerInfo {
     }
 }
 
-pub async fn convert<S: Store>(
+pub struct DockerImageToFlist {
     meta: Writer,
-    store: S,
-    image_name: &str,
+    image_name: String,
     credentials: Option<DockerCredentials>,
-) -> Result<()> {
-    #[cfg(unix)]
-    let docker = Docker::connect_with_socket_defaults().context("failed to create docker")?;
+    docker_tmp_dir: TempDir,
+}
 
-    let container_name = Uuid::new_v4().to_string();
+impl DockerImageToFlist {
+    pub fn new(
+        meta: Writer,
+        image_name: String,
+        credentials: Option<DockerCredentials>,
+        docker_tmp_dir: TempDir,
+    ) -> Self {
+        DockerImageToFlist {
+            meta,
+            image_name,
+            credentials,
+            docker_tmp_dir,
+        }
+    }
 
-    let docker_tmp_dir = tempdir::TempDir::new(&container_name)?;
-    let docker_tmp_dir_path = docker_tmp_dir.path();
+    pub fn files_count(&self) -> usize {
+        WalkDir::new(self.docker_tmp_dir.path()).into_iter().count()
+    }
 
-    let docker_info = DockerInfo {
-        image_name: image_name.to_owned(),
-        container_name,
-        docker,
-    };
+    pub async fn prepare(&mut self) -> Result<()> {
+        #[cfg(unix)]
+        let docker = Docker::connect_with_socket_defaults().context("failed to create docker")?;
 
-    extract_image(
-        &docker_info.docker,
-        &docker_info.image_name,
-        &docker_info.container_name,
-        docker_tmp_dir_path,
-        credentials,
-    )
-    .await
-    .context("failed to extract docker image to a directory")?;
-    log::info!(
-        "docker image '{}' is extracted successfully",
-        docker_info.image_name
-    );
+        let container_file =
+            Path::file_stem(self.docker_tmp_dir.path()).expect("failed to get directory name");
+        let container_name = container_file
+            .to_str()
+            .expect("failed to get container name")
+            .to_owned();
 
-    rfs::pack(meta, store, docker_tmp_dir_path, true)
+        let docker_info = DockerInfo {
+            image_name: self.image_name.to_owned(),
+            container_name,
+            docker,
+        };
+
+        extract_image(
+            &docker_info.docker,
+            &docker_info.image_name,
+            &docker_info.container_name,
+            self.docker_tmp_dir.path(),
+            self.credentials.clone(),
+        )
+        .await
+        .context("failed to extract docker image to a directory")?;
+        log::info!(
+            "docker image '{}' is extracted successfully",
+            docker_info.image_name
+        );
+
+        Ok(())
+    }
+
+    pub async fn pack<S: Store>(&mut self, store: S, sender: Option<Sender<u32>>) -> Result<()> {
+        rfs::pack(
+            self.meta.clone(),
+            store,
+            &self.docker_tmp_dir.path(),
+            true,
+            sender,
+        )
         .await
         .context("failed to pack flist")?;
 
-    log::info!("flist has been created successfully");
-    Ok(())
+        log::info!("flist has been created successfully");
+        Ok(())
+    }
+
+    pub async fn convert<S: Store>(&mut self, store: S, sender: Option<Sender<u32>>) -> Result<()> {
+        self.prepare().await?;
+        self.pack(store, sender).await?;
+
+        Ok(())
+    }
 }
 
 async fn extract_image(
@@ -202,37 +244,37 @@ async fn container_boot(
     let mut env: HashMap<String, String> = HashMap::new();
     let mut cwd = String::from("/");
 
-    let cmd = container_config.cmd.unwrap();
+    let cmd = container_config.cmd.expect("failed to get cmd configs");
 
-    if container_config.entrypoint.is_some() {
-        let entrypoint = container_config.entrypoint.unwrap();
-        command = (entrypoint.first().unwrap()).to_string();
+    if let Some(entrypoint) = container_config.entrypoint {
+        command = (entrypoint.first().expect("failed to get first entrypoint")).to_string();
 
         if entrypoint.len() > 1 {
-            let (_, entries) = entrypoint.split_first().unwrap();
+            let (_, entries) = entrypoint
+                .split_first()
+                .expect("failed to split entrypoint");
             args = entries.to_vec();
         } else {
             args = cmd;
         }
     } else {
-        command = (cmd.first().unwrap()).to_string();
-        let (_, entries) = cmd.split_first().unwrap();
+        command = (cmd.first().expect("failed to get first cmd")).to_string();
+        let (_, entries) = cmd.split_first().expect("failed to split cmd");
         args = entries.to_vec();
     }
 
-    if container_config.env.is_some() {
-        for entry in container_config.env.unwrap().iter() {
-            let mut split = entry.split('=');
-            env.insert(
-                split.next().unwrap().to_string(),
-                split.next().unwrap().to_string(),
-            );
+    if let Some(envs) = container_config.env {
+        for entry in envs.iter() {
+            if let Some((key, value)) = entry.split_once('=') {
+                env.insert(key.to_string(), value.to_string());
+            }
         }
     }
 
-    let working_dir = container_config.working_dir.unwrap();
-    if !working_dir.is_empty() {
-        cwd = working_dir;
+    if let Some(ref working_dir) = container_config.working_dir {
+        if !working_dir.is_empty() {
+            cwd = working_dir.to_string();
+        }
     }
 
     let metadata = json!({

docker2fl/src/main.rs

@@ -3,6 +3,7 @@ use bollard::auth::DockerCredentials;
 use clap::{ArgAction, Parser};
 use rfs::fungi;
 use rfs::store::parse_router;
+use uuid::Uuid;
 
 mod docker2fl;
 
@@ -87,7 +88,13 @@ async fn main() -> Result<()> {
     let meta = fungi::Writer::new(&fl_name, true).await?;
     let store = parse_router(&opts.store).await?;
 
-    let res = docker2fl::convert(meta, store, &docker_image, credentials).await;
+    let container_name = Uuid::new_v4().to_string();
+    let docker_tmp_dir =
+        tempdir::TempDir::new(&container_name).expect("failed to create tmp directory");
+
+    let mut docker_to_fl =
+        docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir);
+    let res = docker_to_fl.convert(store, None).await;
 
     // remove the file created with the writer if fl creation failed
     if res.is_err() {

fl-server/Cargo.toml

@@ -47,3 +47,4 @@ utoipa = { version = "4", features = ["axum_extras"] }
 utoipa-swagger-ui = { version = "7", features = ["axum"] }
 thiserror = "1.0.63"
 hostname-validator = "1.1.1"
+walkdir = "2.5.0"

fl-server/src/config.rs

@@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
     fs,
+    path::PathBuf,
     sync::{Arc, Mutex},
 };
 use utoipa::ToSchema;
@@ -20,6 +21,7 @@ pub struct Job {
 #[derive(ToSchema)]
 pub struct AppState {
     pub jobs_state: Mutex<HashMap<String, handlers::FlistState>>,
+    pub flists_progress: Mutex<HashMap<PathBuf, f32>>,
     pub db: Arc<dyn DB>,
     pub config: Config,
 }