Merge branch 'master' of github.com:threefoldtech/rfs into developmen…

…t_preview_flist

.github/workflows/release.yaml

@@ -31,6 +31,9 @@ jobs:
       - name: Strip
         run: |
           strip target/x86_64-unknown-linux-musl/release/rfs
+      - name: Strip
+        run: |
+          strip target/x86_64-unknown-linux-musl/release/docker2fl
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -41,8 +44,8 @@ jobs:
           release_name: Release ${{ github.ref }}
           draft: false
           prerelease: false
-      - name: Upload Release Asset
-        id: upload-release-asset
+      - name: Upload Release Asset for RFS
+        id: upload-release-asset-rfs
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -51,3 +54,14 @@ jobs:
           asset_path: target/x86_64-unknown-linux-musl/release/rfs
           asset_name: rfs
           asset_content_type: application/x-pie-executable
+
+      - name: Upload Release Asset for docker2fl
+        id: upload-release-asset-docker2fl
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/x86_64-unknown-linux-musl/release/docker2fl
+          asset_name: docker2fl
+          asset_content_type: application/x-pie-executable

.gitignore

@@ -2,3 +2,5 @@
 /tests/*.flist.d
 result
 .direnv/
+fl-server/flists
+fl-server/config.toml

Cargo.toml

@@ -3,8 +3,8 @@ resolver = "2"
 
 members = [
     "rfs",
-		"docker2fl",
-		"fl-server"
+    "docker2fl",
+    "fl-server"
 ]
 
 [profile.release]

Dockerfile

@@ -1,13 +1,23 @@
-FROM rust:slim
+FROM rust:slim as builder
+
+WORKDIR /src
+
+COPY fl-server /src/fl-server
+COPY rfs /src/rfs
+COPY docker2fl /src/docker2fl
+COPY Cargo.toml .
+COPY Cargo.lock .
+COPY config.toml .
 
 RUN apt-get update && apt-get install curl build-essential libssl-dev musl-tools -y
+RUN rustup target add x86_64-unknown-linux-musl
+RUN cargo build --release --bin fl-server --target=x86_64-unknown-linux-musl
 
-WORKDIR /myapp
+FROM alpine:3.19
 
-COPY . .
+WORKDIR /app
 
-RUN rustup target add x86_64-unknown-linux-musl
-RUN cargo build --release --target=x86_64-unknown-linux-musl
+COPY --from=builder /src/target/x86_64-unknown-linux-musl/release/fl-server .
+COPY --from=builder /src/config.toml .
 
-CMD ["/myapp/target/x86_64-unknown-linux-musl/release/fl-server", "--config-path", "config.toml"]
-EXPOSE 3000
+ENTRYPOINT [ "./fl-server", "--config-path", "config.toml"]

docker2fl/src/docker2fl.rs

@@ -5,6 +5,7 @@ use bollard::container::{
 use bollard::image::{CreateImageOptions, RemoveImageOptions};
 use bollard::Docker;
 use std::sync::mpsc::Sender;
+use tempdir::TempDir;
 use walkdir::WalkDir;
 
 use anyhow::{Context, Result};
@@ -13,7 +14,7 @@ use serde_json::json;
 use std::collections::HashMap;
 use std::default::Default;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::process::Command;
 use tokio_async_drop::tokio_async_drop;
 
@@ -43,41 +44,42 @@ impl Drop for DockerInfo {
     }
 }
 
-#[derive(Clone)]
 pub struct DockerImageToFlist {
     meta: Writer,
     image_name: String,
     credentials: Option<DockerCredentials>,
-    docker_tmp_dir_path: PathBuf,
-    files_count: usize,
+    docker_tmp_dir: TempDir,
 }
 
 impl DockerImageToFlist {
     pub fn new(
         meta: Writer,
         image_name: String,
         credentials: Option<DockerCredentials>,
-        docker_tmp_dir_path: PathBuf,
+        docker_tmp_dir: TempDir,
     ) -> Self {
         DockerImageToFlist {
             meta,
             image_name,
             credentials,
-            docker_tmp_dir_path,
-            files_count: 0,
+            docker_tmp_dir,
         }
     }
 
     pub fn files_count(&self) -> usize {
-        self.files_count
+        WalkDir::new(self.docker_tmp_dir.path()).into_iter().count()
     }
 
     pub async fn prepare(&mut self) -> Result<()> {
         #[cfg(unix)]
         let docker = Docker::connect_with_socket_defaults().context("failed to create docker")?;
 
-        let container_file = Path::file_stem(self.docker_tmp_dir_path.as_path()).unwrap();
-        let container_name = container_file.to_str().unwrap().to_owned();
+        let container_file =
+            Path::file_stem(self.docker_tmp_dir.path()).expect("failed to get directory name");
+        let container_name = container_file
+            .to_str()
+            .expect("failed to get container name")
+            .to_owned();
 
         let docker_info = DockerInfo {
             image_name: self.image_name.to_owned(),
@@ -89,7 +91,7 @@ impl DockerImageToFlist {
             &docker_info.docker,
             &docker_info.image_name,
             &docker_info.container_name,
-            &self.docker_tmp_dir_path,
+            self.docker_tmp_dir.path(),
             self.credentials.clone(),
         )
         .await
@@ -99,18 +101,14 @@ impl DockerImageToFlist {
             docker_info.image_name
         );
 
-        self.files_count = WalkDir::new(self.docker_tmp_dir_path.as_path())
-            .into_iter()
-            .count();
-
         Ok(())
     }
 
     pub async fn pack<S: Store>(&mut self, store: S, sender: Option<Sender<u32>>) -> Result<()> {
         rfs::pack(
             self.meta.clone(),
             store,
-            &self.docker_tmp_dir_path,
+            &self.docker_tmp_dir.path(),
             true,
             sender,
         )
@@ -246,37 +244,37 @@ async fn container_boot(
     let mut env: HashMap<String, String> = HashMap::new();
     let mut cwd = String::from("/");
 
-    let cmd = container_config.cmd.unwrap();
+    let cmd = container_config.cmd.expect("failed to get cmd configs");
 
-    if container_config.entrypoint.is_some() {
-        let entrypoint = container_config.entrypoint.unwrap();
-        command = (entrypoint.first().unwrap()).to_string();
+    if let Some(entrypoint) = container_config.entrypoint {
+        command = (entrypoint.first().expect("failed to get first entrypoint")).to_string();
 
         if entrypoint.len() > 1 {
-            let (_, entries) = entrypoint.split_first().unwrap();
+            let (_, entries) = entrypoint
+                .split_first()
+                .expect("failed to split entrypoint");
             args = entries.to_vec();
         } else {
             args = cmd;
         }
     } else {
-        command = (cmd.first().unwrap()).to_string();
-        let (_, entries) = cmd.split_first().unwrap();
+        command = (cmd.first().expect("failed to get first cmd")).to_string();
+        let (_, entries) = cmd.split_first().expect("failed to split cmd");
         args = entries.to_vec();
     }
 
-    if container_config.env.is_some() {
-        for entry in container_config.env.unwrap().iter() {
-            let mut split = entry.split('=');
-            env.insert(
-                split.next().unwrap().to_string(),
-                split.next().unwrap().to_string(),
-            );
+    if let Some(envs) = container_config.env {
+        for entry in envs.iter() {
+            if let Some((key, value)) = entry.split_once('=') {
+                env.insert(key.to_string(), value.to_string());
+            }
         }
     }
 
-    let working_dir = container_config.working_dir.unwrap();
-    if !working_dir.is_empty() {
-        cwd = working_dir;
+    if let Some(ref working_dir) = container_config.working_dir {
+        if !working_dir.is_empty() {
+            cwd = working_dir.to_string();
+        }
     }
 
     let metadata = json!({

docker2fl/src/main.rs

@@ -85,15 +85,15 @@ async fn main() -> Result<()> {
     });
 
     let fl_name = docker_image.replace([':', '/'], "-") + ".fl";
-    let meta = fungi::Writer::new(&fl_name).await?;
+    let meta = fungi::Writer::new(&fl_name, true).await?;
     let store = parse_router(&opts.store).await?;
 
     let container_name = Uuid::new_v4().to_string();
-    let docker_tmp_dir = tempdir::TempDir::new(&container_name).unwrap();
-    let docker_tmp_dir_path = docker_tmp_dir.path().to_owned();
+    let docker_tmp_dir =
+        tempdir::TempDir::new(&container_name).expect("failed to create tmp directory");
 
     let mut docker_to_fl =
-        docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir_path);
+        docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir);
     let res = docker_to_fl.convert(store, None).await;
 
     // remove the file created with the writer if fl creation failed

docs/README.md

@@ -4,7 +4,7 @@
 
 The idea behind the FL format is to build a full filesystem description that is compact and also easy to use from almost ANY language. The format need to be easy to edit by tools like `rfs` or any other tool.
 
-We decided to eventually use `sqlite`! Yes the `FL` file is just a `sqlite` database that has the following [schema](../schema/schema.sql)
+We decided to eventually use `sqlite`! Yes the `FL` file is just a `sqlite` database that has the following [schema](../rfs/schema/schema.sql)
 
 ## Tables
 
@@ -64,10 +64,10 @@ the `block` table is used to associate data file blocks with files. An `id` fiel
 
 the route table holds routing information for the blobs. It basically describe where to find `blobs` with certain `ids`. The routing is done as following:
 
-> Note routing table is loaded one time when `rfs` is started and
+> Note routing table is loaded one time when `rfs` is started.
 
 - We use the first byte of the blob `id` as the `route key`
-- The `route key`` is then consulted against the routing table
+- The `route key` is then consulted against the routing table
 - While building an `FL` all matching stores are updated with the new blob. This is how the system does replication
 - On `getting` an object, the list of matching routes are tried in random order the first one to return a value is used
 - Note that same range and overlapping ranges are allowed, this is how shards and replications are done.

fl-server/src/config.rs

@@ -29,7 +29,7 @@ pub struct AppState {
 #[derive(Debug, Default, Clone, Deserialize)]
 pub struct Config {
     pub host: String,
-    pub port: usize,
+    pub port: u16,
     pub store_url: Vec<String>,
     pub flist_dir: String,
 
@@ -44,14 +44,7 @@ pub async fn parse_config(filepath: &str) -> Result<Config> {
     let c: Config = toml::from_str(&content).context("failed to convert toml config data")?;
 
     if !hostname_validator::is_valid(&c.host) {
-        return Err(anyhow::Error::msg(format!("host '{}' is invalid", c.host)));
-    }
-
-    if c.port > 65535 {
-        return Err(anyhow::Error::msg(format!(
-            "port '{}' is invalid, must be between [0, 65535]",
-            c.port
-        )));
+        anyhow::bail!("host '{}' is invalid", c.host)
     }
 
     rfs::store::parse_router(&c.store_url)
@@ -60,10 +53,10 @@ pub async fn parse_config(filepath: &str) -> Result<Config> {
     fs::create_dir_all(&c.flist_dir).context("failed to create flists directory")?;
 
     if c.jwt_expire_hours < 1 || c.jwt_expire_hours > 24 {
-        return Err(anyhow::Error::msg(format!(
+        anyhow::bail!(format!(
             "jwt expiry interval in hours '{}' is invalid, must be between [1, 24]",
             c.jwt_expire_hours
-        )));
+        ))
     }
 
     Ok(c)

fl-server/src/db.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
@@ -8,24 +10,27 @@ pub struct User {
 }
 
 pub trait DB: Send + Sync {
-    fn get_user_by_username(&self, username: &str) -> Option<&User>;
+    fn get_user_by_username(&self, username: &str) -> Option<User>;
 }
 
 #[derive(Debug, ToSchema)]
-pub struct VecDB {
-    users: Vec<User>,
+pub struct MapDB {
+    users: HashMap<String, User>,
 }
 
-impl VecDB {
+impl MapDB {
     pub fn new(users: &[User]) -> Self {
         Self {
-            users: users.to_vec(),
+            users: users
+                .iter()
+                .map(|u| (u.username.clone(), u.to_owned()))
+                .collect(),
         }
     }
 }
 
-impl DB for VecDB {
-    fn get_user_by_username(&self, username: &str) -> Option<&User> {
-        self.users.iter().find(|u| u.username == username)
+impl DB for MapDB {
+    fn get_user_by_username(&self, username: &str) -> Option<User> {
+        self.users.get(username).cloned()
     }
 }