Basic proto files parsing is implemented (#19)

* Basic proto files parsing is implemented

* return pbuf.yaml

Makefile

@@ -76,7 +76,7 @@ docker:
 .PHONY: run
 # run
 run:
-	docker-compose -f docker-compose.dev.yml build --no-cache && docker-compose -f docker-compose.dev.yml up --force-recreate -d
+	docker-compose -f docker-compose.dev.yml build && docker-compose -f docker-compose.dev.yml up --force-recreate -d
 
 .PHONY: stop
 # stop

api/pbuf-registry/v1/entities.proto

@@ -17,6 +17,9 @@ message Module {
 
   // The draft tags of the module.
   repeated string draft_tags = 4;
+
+  // Packages that uses in this module
+  repeated string packages = 5;
 }
 
 // ProtoFile is a proto file registered in the registry.

cmd/main.go

@@ -29,9 +29,9 @@ type Launcher struct {
 	mainApp  *kratos.App
 	debugApp *kratos.App
 
-	compactionDaemon background.CompactionDaemon
+	compactionDaemon   background.Daemon
+	protoParsingDaemon background.Daemon
 }
-
 func main() {
 	config.NewLoader().MustLoad()
 
@@ -46,7 +46,8 @@ func main() {
 	defer pool.Close()
 
 	registryRepository := data.NewRegistryRepository(pool, logger)
-	registryServer := server.NewRegistryServer(registryRepository, logger)
+	metadataRepository := data.NewMetadataRepository(pool, logger)
+	registryServer := server.NewRegistryServer(registryRepository, metadataRepository, logger)
 
 	app := kratos.New(
 		kratos.ID(id),
@@ -78,12 +79,12 @@ func main() {
 		mainApp:  app,
 		debugApp: debugApp,
 
-		compactionDaemon: background.NewCompactionDaemon(registryRepository, logger),
+		compactionDaemon:   background.NewCompactionDaemon(registryRepository, logger),
+		protoParsingDaemon: background.NewProtoParsingDaemon(metadataRepository, logger),
 	}
 
 	err = CreateRootCommand(launcher).Execute()
 	if err != nil {
-		logHelper.Errorf("failed to execute command: %v", err)
-		return
+		logHelper.Errorf("failed to run application: %v", err)
 	}
 }

cmd/root.go

@@ -4,7 +4,9 @@ import (
 	"time"
 
 	"github.com/go-co-op/gocron"
+	"github.com/go-kratos/kratos/v2"
 	"github.com/go-kratos/kratos/v2/log"
+	"github.com/pbufio/pbuf-registry/internal/background"
 	"github.com/spf13/cobra"
 )
 
@@ -23,6 +25,7 @@ func CreateRootCommand(launcher *Launcher) *cobra.Command {
 	}
 
 	rootCommand.AddCommand(CreateCompactionDaemon(launcher))
+	rootCommand.AddCommand(CreateProtoParsingDaemon(launcher))
 
 	return rootCommand
 }
@@ -32,33 +35,57 @@ func CreateCompactionDaemon(launcher *Launcher) *cobra.Command {
 		Use:   "compaction",
 		Short: "Run compaction daemon",
 		Run: func(cmd *cobra.Command, args []string) {
-			s := gocron.NewScheduler(time.UTC)
+			runBackgroundDaemon(
+				launcher.config.Daemons.Compaction.CronSchedule,
+				launcher.compactionDaemon,
+				launcher.debugApp,
+			)
+		},
+	}
 
-			// start every hour
-			_, err := s.Cron(launcher.config.Daemons.Compaction.CronSchedule).Do(func() {
-				err := launcher.compactionDaemon.Run()
-				if err != nil {
-					log.Fatalf("failed to run compaction daemon: %v", err)
-				}
-			})
+	return compactionDaemonCommand
+}
 
-			if err != nil {
-				log.Fatalf("failed to create cron job: %v", err)
-			}
+func CreateProtoParsingDaemon(launcher *Launcher) *cobra.Command {
+	protoParsingDaemonCommand := &cobra.Command{
+		Use:   "proto-parsing",
+		Short: "Run proto parsing daemon",
+		Run: func(cmd *cobra.Command, args []string) {
+			runBackgroundDaemon(
+				launcher.config.Daemons.ProtoParsing.CronSchedule,
+				launcher.protoParsingDaemon,
+				launcher.debugApp,
+			)
+		},
+	}
 
-			// start the scheduler
-			s.StartAsync()
+	return protoParsingDaemonCommand
+}
 
-			err = launcher.debugApp.Run()
-			if err != nil {
-				log.Fatalf("failed to run debug app: %v", err)
-			}
+func runBackgroundDaemon(cronSchedule string, daemon background.Daemon, debugApp *kratos.App) {
+	s := gocron.NewScheduler(time.UTC)
 
-			s.Stop()
+	// start every hour
+	_, err := s.Cron(cronSchedule).Do(func() {
+		err := daemon.Run()
+		if err != nil {
+			log.Fatalf("failed to run %s daemon: %v", daemon.Name(), err)
+		}
+	})
 
-			log.Infof("Compaction daemon stopped")
-		},
+	if err != nil {
+		log.Fatalf("failed to create cron job: %v", err)
 	}
 
-	return compactionDaemonCommand
+	// start the scheduler
+	s.StartAsync()
+
+	err = debugApp.Run()
+	if err != nil {
+		log.Fatalf("failed to run debug app: %v", err)
+	}
+
+	s.Stop()
+
+	log.Infof("%s daemon stopped", daemon.Name())
 }

docker-compose.dev.yml

@@ -40,8 +40,6 @@ services:
     depends_on:
       - db
       - pbuf-registry
-    ports:
-      - "8083:8082"
     healthcheck:
       test: wget -O - http://localhost:8082/healthz || exit 1
       interval: 5s
@@ -50,4 +48,20 @@ services:
     environment:
       DATA_DATABASE_DSN: "postgres://pbuf:pbuf@db:5432/pbuf_registry?sslmode=disable"
     command: >
-      sh -c "/app/pbuf-registry compaction"
+      sh -c "/app/pbuf-registry compaction"
+  pbuf-registry-protoparsing:
+    build:
+      context: .
+    restart: always
+    depends_on:
+      - db
+      - pbuf-registry
+    healthcheck:
+      test: wget -O - http://localhost:8082/healthz || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    environment:
+      DATA_DATABASE_DSN: "postgres://pbuf:pbuf@db:5432/pbuf_registry?sslmode=disable"
+    command: >
+      sh -c "/app/pbuf-registry proto-parsing"

docker-compose.yml

@@ -19,7 +19,7 @@ services:
       timeout: 10s
       retries: 5
   pbuf-registry:
-    image: ghcr.io/pbufio/registry:v0.4.0-wip.1
+    image: ghcr.io/pbufio/registry:v0.4.0-wip.2
     restart: always
     depends_on:
       - db
@@ -49,7 +49,7 @@ services:
     command: >
       sh -c "/app/pbuf-migrations && /app/pbuf-registry"
   pbuf-registry-compaction:
-    image: ghcr.io/pbufio/registry:v0.4.0-wip.1
+    image: ghcr.io/pbufio/registry:v0.4.0-wip.2
     restart: always
     depends_on:
       - db
@@ -65,6 +65,23 @@ services:
       DATA_DATABASE_DSN: "postgres://pbuf:pbuf@db:5432/pbuf_registry?sslmode=disable"
     command: >
       sh -c "/app/pbuf-registry compaction"
+  pbuf-registry-protoparser:
+    image: ghcr.io/pbufio/registry:v0.4.0-wip.2
+    restart: always
+    depends_on:
+      - db
+      - pbuf-registry
+    ports:
+      - "127.0.0.1:8084:8082"
+    healthcheck:
+      test: wget -O - http://localhost:8082/healthz || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    environment:
+      DATA_DATABASE_DSN: "postgres://pbuf:pbuf@db:5432/pbuf_registry?sslmode=disable"
+    command: >
+      sh -c "/app/pbuf-registry proto-parsing"
 
 networks:
   internal:

gen/pbuf-registry/v1/registry.swagger.json

@@ -468,6 +468,13 @@
             "type": "string"
           },
           "description": "The draft tags of the module."
+        },
+        "packages": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "title": "Packages that uses in this module"
         }
       },
       "description": "Module is a module registered in the registry."

internal/background/background.go

@@ -0,0 +1,6 @@
+package background
+
+type Daemon interface {
+	Name() string
+	Run() error
+}

internal/background/compaction.go

@@ -7,22 +7,26 @@ import (
 	"github.com/pbufio/pbuf-registry/internal/data"
 )
 
-type CompactionDaemon interface {
-	Run() error
-}
+const (
+	compactionDaemonName = "compaction"
+)
 
 type compactionDaemon struct {
 	registryRepository data.RegistryRepository
 	log                *log.Helper
 }
 
-func NewCompactionDaemon(registryRepository data.RegistryRepository, logger log.Logger) CompactionDaemon {
+func NewCompactionDaemon(registryRepository data.RegistryRepository, logger log.Logger) Daemon {
 	return &compactionDaemon{
 		registryRepository: registryRepository,
 		log:                log.NewHelper(log.With(logger, "module", "background/CompactionDaemon")),
 	}
 }
 
+func (d *compactionDaemon) Name() string {
+	return compactionDaemonName
+}
+
 func (d *compactionDaemon) Run() error {
 	d.log.Infof("Running compaction")