Merge pull request #133 from Maker-Management-Platform/discovery-rework

Rework

.dockerignore

@@ -0,0 +1 @@
+testdata*

core/api/projects/assets/new.go

@@ -10,7 +10,8 @@ import (
 	"github.com/eduardooliveira/stLib/core/data/database"
 	"github.com/eduardooliveira/stLib/core/downloader/tools"
 	"github.com/eduardooliveira/stLib/core/entities"
-	"github.com/eduardooliveira/stLib/core/processing"
+	"github.com/eduardooliveira/stLib/core/processing/initialization"
+	"github.com/eduardooliveira/stLib/core/processing/types"
 	"github.com/eduardooliveira/stLib/core/utils"
 	"github.com/labstack/echo/v4"
 	"gorm.io/gorm"
@@ -56,13 +57,19 @@ func New(c echo.Context) error {
 	defer src.Close()
 	if err = tools.SaveFile(filepath.Join(path, files[0].Filename), src); err != nil {
 		log.Println(err)
-		return c.NoContent(http.StatusInternalServerError)
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
-	processing.EnqueueInitJob(&processing.ProcessableAsset{
+
+	_, err = initialization.NewAssetIniter(&types.ProcessableAsset{
 		Name:    files[0].Filename,
 		Project: project,
 		Origin:  "fs",
-	})
+	}).Init()
+
+	if err != nil {
+		log.Println(err)
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
+	}
 
 	return c.NoContent(http.StatusCreated)
 }

core/api/projects/discoverHandler.go

@@ -6,7 +6,9 @@ import (
 	"net/http"
 
 	"github.com/eduardooliveira/stLib/core/data/database"
-	"github.com/eduardooliveira/stLib/core/processing"
+	"github.com/eduardooliveira/stLib/core/processing/discovery"
+	"github.com/eduardooliveira/stLib/core/processing/initialization"
+	"github.com/eduardooliveira/stLib/core/processing/types"
 	"github.com/labstack/echo/v4"
 	"gorm.io/gorm"
 )
@@ -28,9 +30,14 @@ func discoverHandler(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
 
-	_, err = processing.HandlePath(project.FullPath())
+	_, err = initialization.NewProjectIniter(types.ProcessableProject{
+		Path: project.FullPath(),
+	}).
+		WithAssetDiscoverer(discovery.FlatAssetDiscoverer{}).
+		Init()
+
 	if err != nil {
-		log.Printf("error discovering the project %q: %v\n", project.FullPath(), err)
+		log.Println(err)
 		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
 

core/api/projects/new.go

@@ -11,7 +11,9 @@ import (
 
 	"github.com/eduardooliveira/stLib/core/data/database"
 	"github.com/eduardooliveira/stLib/core/entities"
-	"github.com/eduardooliveira/stLib/core/processing"
+	"github.com/eduardooliveira/stLib/core/processing/discovery"
+	"github.com/eduardooliveira/stLib/core/processing/initialization"
+	"github.com/eduardooliveira/stLib/core/processing/types"
 	"github.com/eduardooliveira/stLib/core/utils"
 	"github.com/labstack/echo/v4"
 )
@@ -83,11 +85,16 @@ func new(c echo.Context) error {
 		}
 
 	}
-	var project *entities.Project
-	if project, err = processing.HandlePath(projectFolder); err != nil {
-		log.Printf("error loading the project %q: %v\n", path, err)
+
+	pp, err := initialization.NewProjectIniter(types.ProcessableProject{
+		Path: projectFolder,
+	}).WithAssetDiscoverer(discovery.FlatAssetDiscoverer{}).
+		Init()
+	if err != nil {
+		log.Println(err)
 		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
+	project := pp.Project
 
 	project.Description = createProject.Description
 	project.Tags = createProject.Tags

core/api/system/endpoints.go

@@ -1,6 +1,7 @@
 package system
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -67,7 +68,12 @@ func saveSettings(c echo.Context) error {
 }
 
 func runDiscovery(c echo.Context) error {
-	go processing.Run(runtime.Cfg.Library.Path)
+	go func() {
+		err := processing.ProcessFolder(context.Background(), runtime.Cfg.Library.Path)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
 	return c.NoContent(http.StatusOK)
 }
 

core/api/tempfiles/endpoints.go

@@ -9,7 +9,8 @@ import (
 	"github.com/duke-git/lancet/v2/maputil"
 	"github.com/eduardooliveira/stLib/core/data/database"
 	models "github.com/eduardooliveira/stLib/core/entities"
-	"github.com/eduardooliveira/stLib/core/processing"
+	"github.com/eduardooliveira/stLib/core/processing/initialization"
+	"github.com/eduardooliveira/stLib/core/processing/types"
 	"github.com/eduardooliveira/stLib/core/runtime"
 	"github.com/eduardooliveira/stLib/core/state"
 	"github.com/eduardooliveira/stLib/core/utils"
@@ -60,11 +61,16 @@ func move(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
 	}
 
-	processing.EnqueueInitJob(&processing.ProcessableAsset{
+	_, err = initialization.NewAssetIniter(&types.ProcessableAsset{
 		Name:    tempFile.Name,
 		Project: project,
 		Origin:  "fs",
-	})
+	}).Init()
+
+	if err != nil {
+		log.Println(err)
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
+	}
 
 	delete(state.TempFiles, uuid)
 	return c.NoContent(http.StatusOK)

core/downloader/makerworld/makerworld.go

@@ -19,6 +19,7 @@ import (
 	"github.com/eduardooliveira/stLib/core/data/database"
 	"github.com/eduardooliveira/stLib/core/downloader/tools"
 	"github.com/eduardooliveira/stLib/core/entities"
+	"github.com/eduardooliveira/stLib/core/processing/types"
 	"github.com/eduardooliveira/stLib/core/utils"
 	"golang.org/x/net/html"
 )
@@ -73,12 +74,19 @@ func Fetch(urlString string, cookies []*http.Cookie, userAgent string) error {
 		return err
 	}
 
-	assets := make([]*entities.ProjectAsset, 0)
+	assets := make([]*types.ProcessableAsset, 0)
 	as, err := mwc.fetchCover()
 	if err != nil {
 		log.Println("error fetching cover")
 		return err
 	}
+
+	for _, a := range as {
+		if a.Asset != nil {
+			project.DefaultImageID = a.Asset.ID
+		}
+	}
+
 	assets = append(assets, as...)
 
 	as, err = mwc.fetchModels()
@@ -88,75 +96,83 @@ func Fetch(urlString string, cookies []*http.Cookie, userAgent string) error {
 	}
 	assets = append(assets, as...)
 
-	err = mwc.fetchInstances()
+	_, err = mwc.fetchInstances()
 	if err != nil {
 		log.Println("error fetching models")
 		return err
 	}
 
-	err = mwc.fetchPictures()
+	as, err = mwc.fetchPictures()
 	if err != nil {
 		log.Println("error fetching pictures")
 		return err
 	}
+	assets = append(assets, as...)
+
+	if project.DefaultImageID == "" {
+		for _, a := range assets {
+			if a.Asset != nil && a.Asset.AssetType == "image" {
+				project.DefaultImageID = a.Asset.ID
+				break
+			}
+		}
+	}
 
 	return database.InsertProject(project)
 }
 
-func (mwc *mwClient) fetchCover() (assets []*entities.ProjectAsset, err error) {
+func (mwc *mwClient) fetchCover() ([]*types.ProcessableAsset, error) {
 	req, err := http.NewRequest("GET", mwc.metadata.Props.PageProps.Design.CoverURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Add("User-Agent", mwc.userAgent)
 
-	assets = make([]*entities.ProjectAsset, 0)
-	err = tools.DownloadAsset(path.Base(mwc.metadata.Props.PageProps.Design.CoverURL), mwc.project, mwc.client, req)
-	if err != nil {
-		log.Println("Error fetchig cover, skiping: ", err)
-		return
-	}
-
-	return
+	return tools.DownloadAsset(path.Base(mwc.metadata.Props.PageProps.Design.CoverURL), mwc.project, mwc.client, req)
 }
 
-func (mwc *mwClient) fetchPictures() (err error) {
+func (mwc *mwClient) fetchPictures() ([]*types.ProcessableAsset, error) {
+	assets := make([]*types.ProcessableAsset, 0)
 	for _, p := range mwc.metadata.Props.PageProps.Design.DesignExtension.DesignPictures {
 		req, err := http.NewRequest("GET", p.URL, nil)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		req.Header.Add("User-Agent", mwc.userAgent)
 
-		err = tools.DownloadAsset(p.Name, mwc.project, mwc.client, req)
+		a, err := tools.DownloadAsset(p.Name, mwc.project, mwc.client, req)
 		if err != nil {
 			log.Println("Error fetchig image, skiping: ", err)
 			continue
 		}
+		assets = append(assets, a...)
 	}
 
-	return
+	return assets, nil
 }
 
-func (mwc *mwClient) fetchModels() (assets []*entities.ProjectAsset, err error) {
+func (mwc *mwClient) fetchModels() ([]*types.ProcessableAsset, error) {
+	assets := make([]*types.ProcessableAsset, 0)
 	for _, m := range mwc.metadata.Props.PageProps.Design.DesignExtension.ModelFiles {
 		req, err := http.NewRequest("GET", m.ModelURL, nil)
 		if err != nil {
 			return nil, err
 		}
 		req.Header.Add("User-Agent", mwc.userAgent)
 
-		err = tools.DownloadAsset(m.ModelName, mwc.project, mwc.client, req)
+		a, err := tools.DownloadAsset(m.ModelName, mwc.project, mwc.client, req)
 		if err != nil {
 			log.Println("Error fetchig model, skiping: ", err)
 			continue
 		}
+		assets = append(assets, a...)
 	}
 
-	return
+	return assets, nil
 }
 
-func (mwc *mwClient) fetchInstances() (err error) {
+func (mwc *mwClient) fetchInstances() ([]*types.ProcessableAsset, error) {
+	assets := make([]*types.ProcessableAsset, 0)
 
 	for _, m := range mwc.metadata.Props.PageProps.Design.Instances {
 		sl := rand.Intn(6000-3000) + 3000
@@ -169,7 +185,7 @@ func (mwc *mwClient) fetchInstances() (err error) {
 		}
 		req, err := http.NewRequest("GET", mfData.URL, nil)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		req.Header.Add("User-Agent", mwc.userAgent)
 
@@ -180,14 +196,16 @@ func (mwc *mwClient) fetchInstances() (err error) {
 		sl = rand.Intn(6000-3000) + 3000
 		log.Println("sleeping: ", sl)
 		time.Sleep(time.Duration(sl) * time.Millisecond)
-		err = tools.DownloadAsset(name, mwc.project, mwc.client, req)
+
+		a, err := tools.DownloadAsset(name, mwc.project, mwc.client, req)
 		if err != nil {
 			log.Println("Failed to download 3MF File, skipping")
 			continue
 		}
+		assets = append(assets, a...)
 	}
 
-	return
+	return assets, nil
 }
 
 func (mwc *mwClient) fetch3MFData(id int) (*mf, error) {