Merge branch 'master' of github.com:threefoldtech/rfs into developmen…

…t_support_conversion_progress

.github/workflows/release.yaml

@@ -31,6 +31,9 @@ jobs:
       - name: Strip
         run: |
           strip target/x86_64-unknown-linux-musl/release/rfs
+      - name: Strip
+        run: |
+          strip target/x86_64-unknown-linux-musl/release/docker2fl
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -41,8 +44,8 @@ jobs:
           release_name: Release ${{ github.ref }}
           draft: false
           prerelease: false
-      - name: Upload Release Asset
-        id: upload-release-asset
+      - name: Upload Release Asset for RFS
+        id: upload-release-asset-rfs
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -51,3 +54,14 @@ jobs:
           asset_path: target/x86_64-unknown-linux-musl/release/rfs
           asset_name: rfs
           asset_content_type: application/x-pie-executable
+
+      - name: Upload Release Asset for docker2fl
+        id: upload-release-asset-docker2fl
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/x86_64-unknown-linux-musl/release/docker2fl
+          asset_name: docker2fl
+          asset_content_type: application/x-pie-executable

.gitignore

@@ -2,3 +2,5 @@
 /tests/*.flist.d
 result
 .direnv/
+fl-server/flists
+fl-server/config.toml

Cargo.toml

@@ -3,8 +3,8 @@ resolver = "2"
 
 members = [
     "rfs",
-		"docker2fl",
-		"fl-server"
+    "docker2fl",
+    "fl-server"
 ]
 
 [profile.release]

Dockerfile

@@ -1,13 +1,23 @@
-FROM rust:slim
+FROM rust:slim as builder
+
+WORKDIR /src
+
+COPY fl-server /src/fl-server
+COPY rfs /src/rfs
+COPY docker2fl /src/docker2fl
+COPY Cargo.toml .
+COPY Cargo.lock .
+COPY config.toml .
 
 RUN apt-get update && apt-get install curl build-essential libssl-dev musl-tools -y
+RUN rustup target add x86_64-unknown-linux-musl
+RUN cargo build --release --bin fl-server --target=x86_64-unknown-linux-musl
 
-WORKDIR /myapp
+FROM alpine:3.19
 
-COPY . .
+WORKDIR /app
 
-RUN rustup target add x86_64-unknown-linux-musl
-RUN cargo build --release --target=x86_64-unknown-linux-musl
+COPY --from=builder /src/target/x86_64-unknown-linux-musl/release/fl-server .
+COPY --from=builder /src/config.toml .
 
-CMD ["/myapp/target/x86_64-unknown-linux-musl/release/fl-server", "--config-path", "config.toml"]
-EXPOSE 3000
+ENTRYPOINT [ "./fl-server", "--config-path", "config.toml"]

docker2fl/src/main.rs

@@ -85,7 +85,7 @@ async fn main() -> Result<()> {
     });
 
     let fl_name = docker_image.replace([':', '/'], "-") + ".fl";
-    let meta = fungi::Writer::new(&fl_name).await?;
+    let meta = fungi::Writer::new(&fl_name, true).await?;
     let store = parse_router(&opts.store).await?;
 
     let container_name = Uuid::new_v4().to_string();

docs/README.md

@@ -4,7 +4,7 @@
 
 The idea behind the FL format is to build a full filesystem description that is compact and also easy to use from almost ANY language. The format need to be easy to edit by tools like `rfs` or any other tool.
 
-We decided to eventually use `sqlite`! Yes the `FL` file is just a `sqlite` database that has the following [schema](../schema/schema.sql)
+We decided to eventually use `sqlite`! Yes the `FL` file is just a `sqlite` database that has the following [schema](../rfs/schema/schema.sql)
 
 ## Tables
 
@@ -64,10 +64,10 @@ the `block` table is used to associate data file blocks with files. An `id` fiel
 
 the route table holds routing information for the blobs. It basically describe where to find `blobs` with certain `ids`. The routing is done as following:
 
-> Note routing table is loaded one time when `rfs` is started and
+> Note routing table is loaded one time when `rfs` is started.
 
 - We use the first byte of the blob `id` as the `route key`
-- The `route key`` is then consulted against the routing table
+- The `route key` is then consulted against the routing table
 - While building an `FL` all matching stores are updated with the new blob. This is how the system does replication
 - On `getting` an object, the list of matching routes are tried in random order the first one to return a value is used
 - Note that same range and overlapping ranges are allowed, this is how shards and replications are done.

fl-server/src/config.rs

@@ -29,7 +29,7 @@ pub struct AppState {
 #[derive(Debug, Default, Clone, Deserialize)]
 pub struct Config {
     pub host: String,
-    pub port: usize,
+    pub port: u16,
     pub store_url: Vec<String>,
     pub flist_dir: String,
 
@@ -44,14 +44,7 @@ pub async fn parse_config(filepath: &str) -> Result<Config> {
     let c: Config = toml::from_str(&content).context("failed to convert toml config data")?;
 
     if !hostname_validator::is_valid(&c.host) {
-        return Err(anyhow::Error::msg(format!("host '{}' is invalid", c.host)));
-    }
-
-    if c.port > 65535 {
-        return Err(anyhow::Error::msg(format!(
-            "port '{}' is invalid, must be between [0, 65535]",
-            c.port
-        )));
+        anyhow::bail!("host '{}' is invalid", c.host)
     }
 
     rfs::store::parse_router(&c.store_url)
@@ -60,10 +53,10 @@ pub async fn parse_config(filepath: &str) -> Result<Config> {
     fs::create_dir_all(&c.flist_dir).context("failed to create flists directory")?;
 
     if c.jwt_expire_hours < 1 || c.jwt_expire_hours > 24 {
-        return Err(anyhow::Error::msg(format!(
+        anyhow::bail!(format!(
             "jwt expiry interval in hours '{}' is invalid, must be between [1, 24]",
             c.jwt_expire_hours
-        )));
+        ))
     }
 
     Ok(c)

fl-server/src/db.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
@@ -8,24 +10,27 @@ pub struct User {
 }
 
 pub trait DB: Send + Sync {
-    fn get_user_by_username(&self, username: &str) -> Option<&User>;
+    fn get_user_by_username(&self, username: &str) -> Option<User>;
 }
 
 #[derive(Debug, ToSchema)]
-pub struct VecDB {
-    users: Vec<User>,
+pub struct MapDB {
+    users: HashMap<String, User>,
 }
 
-impl VecDB {
+impl MapDB {
     pub fn new(users: &[User]) -> Self {
         Self {
-            users: users.to_vec(),
+            users: users
+                .iter()
+                .map(|u| (u.username.clone(), u.to_owned()))
+                .collect(),
         }
     }
 }
 
-impl DB for VecDB {
-    fn get_user_by_username(&self, username: &str) -> Option<&User> {
-        self.users.iter().find(|u| u.username == username)
+impl DB for MapDB {
+    fn get_user_by_username(&self, username: &str) -> Option<User> {
+        self.users.get(username).cloned()
     }
 }

fl-server/src/handlers.rs

@@ -115,17 +115,16 @@ pub async fn create_flist_handler(
         return Err(ResponseError::Conflict("flist already exists".to_string()));
     }
 
-    let created = fs::create_dir_all(&username_dir);
-    if created.is_err() {
+    if let Err(err) = fs::create_dir_all(&username_dir) {
         log::error!(
             "failed to create user flist directory `{:?}` with error {:?}",
             &username_dir,
-            created.err()
+            err
         );
         return Err(ResponseError::InternalServerError);
     }
 
-    let meta = match Writer::new(&fl_path).await {
+    let meta = match Writer::new(&fl_path, true).await {
         Ok(writer) => writer,
         Err(err) => {
             log::error!(

fl-server/src/main.rs

@@ -72,7 +72,7 @@ async fn app() -> Result<()> {
         .await
         .context("failed to parse config file")?;
 
-    let db = Arc::new(db::VecDB::new(&config.users));
+    let db = Arc::new(db::MapDB::new(&config.users.clone()));
 
     let app_state = Arc::new(config::AppState {
         jobs_state: Mutex::new(HashMap::new()),

fl-server/src/response.rs

@@ -56,19 +56,15 @@ impl IntoResponse for ResponseResult {
             ResponseResult::SignedIn(token) => {
                 (StatusCode::CREATED, Json(serde_json::json!(token))).into_response()
             }
-            ResponseResult::FlistCreated(job) => {
-                (StatusCode::CREATED, Json(serde_json::json!(job))).into_response()
-            }
+            ResponseResult::FlistCreated(job) => (StatusCode::CREATED, Json(job)).into_response(),
             ResponseResult::FlistState(flist_state) => (
                 StatusCode::OK,
                 Json(serde_json::json!({
                     "flist_state": flist_state
                 })),
             )
                 .into_response(),
-            ResponseResult::Flists(flists) => {
-                (StatusCode::OK, Json(serde_json::json!(flists))).into_response()
-            }
+            ResponseResult::Flists(flists) => (StatusCode::OK, Json(flists)).into_response(),
         }
     }
 }

fl-server/src/serve_flists.rs

@@ -4,7 +4,7 @@ use axum::{
     response::{Html, Response},
 };
 use serde::Serialize;
-use std::{path::PathBuf, sync::Arc};
+use std::{io::Error, path::PathBuf, sync::Arc};
 use tokio::io;
 use tower::util::ServiceExt;
 use tower_http::services::ServeDir;
@@ -32,22 +32,16 @@ pub async fn serve_flists(
             let status = res.status();
             match status {
                 StatusCode::NOT_FOUND => {
-                    let path = path.trim_start_matches('/');
-                    let path = percent_decode(path.as_ref()).decode_utf8_lossy();
-
-                    let mut full_path = PathBuf::new();
-
-                    // validate
-                    for seg in path.split('/') {
-                        if seg.starts_with("..") || seg.contains('\\') {
+                    let full_path = match validate_path(&path) {
+                        Ok(p) => p,
+                        Err(_) => {
                             return Err(ErrorTemplate {
                                 err: ResponseError::BadRequest("invalid path".to_string()),
                                 cur_path: path.to_string(),
                                 message: "invalid path".to_owned(),
                             });
                         }
-                        full_path.push(seg);
-                    }
+                    };
 
                     let cur_path = std::path::Path::new(&full_path);
 
@@ -85,6 +79,23 @@ pub async fn serve_flists(
     };
 }
 
+fn validate_path(path: &str) -> io::Result<PathBuf> {
+    let path = path.trim_start_matches('/');
+    let path = percent_decode(path.as_ref()).decode_utf8_lossy();
+
+    let mut full_path = PathBuf::new();
+
+    // validate
+    for seg in path.split('/') {
+        if seg.starts_with("..") || seg.contains('\\') {
+            return Err(Error::other("invalid path"));
+        }
+        full_path.push(seg);
+    }
+
+    Ok(full_path)
+}
+
 pub async fn visit_dir_one_level<P: AsRef<std::path::Path>>(
     path: P,
     state: &Arc<config::AppState>,

frontend/.env

@@ -0,0 +1 @@
+VITE_API_URL="http://localhost:4000"

frontend/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

frontend/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["Vue.volar"]
+}

frontend/Dockerfile

@@ -0,0 +1,13 @@
+# build stage
+FROM node:lts-alpine as build-stage
+WORKDIR /app
+COPY package*.json ./
+RUN npm install
+COPY . .
+RUN npm run build
+
+# production stage
+FROM nginx:stable-alpine as production-stage
+COPY --from=build-stage /app/dist /usr/share/nginx/html
+EXPOSE 80
+CMD ["nginx", "-g", "daemon off;"]