update: 通知同期システムを追加 (#121)

cmd/rikka/main.go

@@ -74,6 +74,7 @@ func init() {
 		&entity.Problem{},
 		&entity.Answer{},
 		&entity.Attachment{},
+		&entity.Notice{},
 	)
 
 	store, err = redis.NewStore(

cmd/ritto/.gitignore

@@ -1 +1,2 @@
-config.yaml
+config.yaml
+config.yaml.prod

cmd/ritto/config.go

@@ -30,11 +30,12 @@ type Rikka struct {
 }
 
 type GrowiConfig struct {
-	Url      string `yaml:"url"`
-	Token    string `yaml:"token"`
-	Path     string `yaml:"path"`
-	Username string `yaml:"username"`
-	Password string `yaml:"password"`
+	Url         string `yaml:"url"`
+	Token       string `yaml:"token"`
+	ProblemPath string `yaml:"problemPath"`
+	NoticePath  string `yaml:"noticePath"`
+	Username    string `yaml:"username"`
+	Password    string `yaml:"password"`
 }
 
 func (c *MariaDBConfig) getDSN() string {

cmd/ritto/main.go

@@ -86,6 +86,8 @@ func main() {
 	growiSessionCookieRepo := rc.NewGrowiSessionCookieRepository(redisClient)
 	problemWithSyncTimeRepo := rc.NewProblemWithSyncTimeRepository(redisClient)
 	problemRepo := mariadb.NewProblemRepository(db)
+	noticeRepo := mariadb.NewNoticeRepository(db)
+	noticeWithSyncTimeRepo := rc.NewNoticeWithSyncTimeRepository(redisClient)
 
 	option := &growi_client.GrowiClientOption{
 		Username:    config.Growi.Username,
@@ -115,16 +117,24 @@ func main() {
 	)
 	growiProblemSyncService := service.NewGrowiProblemSyncService(
 		client,
-		config.Growi.Path,
+		config.Growi.ProblemPath,
 		config.Rikka.AuthorId,
 		problemWithSyncTimeRepo,
 		problemRepo,
 	)
+	growiNoticeSyncService := service.NewGrowiNoticeSyncService(
+		client,
+		config.Growi.NoticePath,
+		noticeWithSyncTimeRepo,
+		noticeRepo,
+	)
 
 	err = growiClientInitService.Init(ctx)
 	// TODO(k-shir0): エラー処理追加
 	err = growiProblemSyncService.Sync(ctx)
 	// TODO(k-shir0): エラー処理追加
+	err = growiNoticeSyncService.Sync(ctx)
+	// TODO(k-shir0): エラー処理追加
 	if err != nil {
 		log.Fatal(err)
 	}

pkg/entity/notice.go

@@ -0,0 +1,19 @@
+package entity
+
+import (
+	"time"
+)
+
+type Notice struct {
+	Base
+
+	SourceId string `json:"source_id"`
+	Title    string `json:"title"`
+	Body     string `json:"body,omitempty"`
+	Draft    bool   `json:"draft"`
+}
+
+type NoticeWithSyncTime struct {
+	Notice
+	UpdatedAt time.Time
+}

pkg/entity/notice_front_matter.go

@@ -0,0 +1,16 @@
+package entity
+
+import "errors"
+
+type NoticeFrontMatter struct {
+	Title string `yaml:"title"`
+	Draft bool   `yaml:"draft"`
+}
+
+func (n *NoticeFrontMatter) Validate() error {
+	if n.Title == "" {
+		return errors.New("title must not be empty")
+	}
+
+	return nil
+}

pkg/entity/front_matter.go → pkg/entity/problem_front_matter.go

@@ -1,6 +1,6 @@
 package entity
 
-type FrontMatter struct {
+type ProblemFrontMatter struct {
 	Code              string  `yaml:"code"`
 	Title             string  `yaml:"title"`
 	Point             uint    `yaml:"point"`

pkg/repository/mariadb/notice_repository.go

@@ -0,0 +1,45 @@
+package mariadb
+
+import (
+	"github.com/google/uuid"
+	"github.com/ictsc/ictsc-rikka/pkg/entity"
+	"gorm.io/gorm"
+)
+
+type NoticeRepository struct {
+	db *gorm.DB
+}
+
+func NewNoticeRepository(db *gorm.DB) *NoticeRepository {
+	return &NoticeRepository{
+		db: db,
+	}
+}
+
+func (r *NoticeRepository) Create(notice *entity.Notice) (*entity.Notice, error) {
+	err := r.db.Create(notice).Error
+	if err != nil {
+		return nil, err
+	}
+	return r.FindByID(notice.ID)
+}
+
+func (r *NoticeRepository) GetAll() ([]*entity.Notice, error) {
+	notice := make([]*entity.Notice, 0)
+	err := r.db.Find(&notice).Error
+	return notice, err
+}
+
+func (r *NoticeRepository) FindByID(id uuid.UUID) (*entity.Notice, error) {
+	res := &entity.Notice{}
+	err := r.db.First(res, id).Error
+	return res, err
+}
+
+func (r *NoticeRepository) Update(notice *entity.Notice) (*entity.Notice, error) {
+	err := r.db.Save(notice).Error
+	if err != nil {
+		return nil, err
+	}
+	return r.FindByID(notice.ID)
+}

pkg/repository/notice_repository.go

@@ -0,0 +1,13 @@
+package repository
+
+import (
+	"github.com/google/uuid"
+	"github.com/ictsc/ictsc-rikka/pkg/entity"
+)
+
+type NoticeRepository interface {
+	Create(notice *entity.Notice) (*entity.Notice, error)
+	GetAll() ([]*entity.Notice, error)
+	FindByID(id uuid.UUID) (*entity.Notice, error)
+	Update(notice *entity.Notice) (*entity.Notice, error)
+}

pkg/repository/notice_with_sync_time.go

@@ -0,0 +1,11 @@
+package repository
+
+import (
+	"context"
+	"github.com/ictsc/ictsc-rikka/pkg/entity"
+)
+
+type NoticeWithSyncTimeRepository interface {
+	Set(context.Context, string, entity.NoticeWithSyncTime) error
+	Get(context.Context, string) (*entity.NoticeWithSyncTime, error)
+}

pkg/repository/rc/notice_with_sync_time.go

@@ -0,0 +1,38 @@
+package rc
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/go-redis/redis/v8"
+	"github.com/ictsc/ictsc-rikka/pkg/entity"
+)
+
+type NoticeWithSyncTimeRepository struct {
+	rc *redis.Client
+}
+
+func NewNoticeWithSyncTimeRepository(rc *redis.Client) *NoticeWithSyncTimeRepository {
+	return &NoticeWithSyncTimeRepository{rc: rc}
+}
+
+func (r *NoticeWithSyncTimeRepository) Set(ctx context.Context, path string, noticeWithInfo entity.NoticeWithSyncTime) error {
+	jsonBytes, err := json.Marshal(noticeWithInfo)
+	if err != nil {
+		return err
+	}
+
+	err = r.rc.Set(ctx, path, jsonBytes, 0).Err()
+	return err
+}
+
+func (r *NoticeWithSyncTimeRepository) Get(ctx context.Context, path string) (*entity.NoticeWithSyncTime, error) {
+	noticeWithInfo, err := r.rc.Get(ctx, path).Result()
+	if err != nil {
+		return nil, err
+	}
+
+	noticeWithInfoEntity := entity.NoticeWithSyncTime{}
+	err = json.Unmarshal([]byte(noticeWithInfo), &noticeWithInfoEntity)
+
+	return &noticeWithInfoEntity, err
+}

pkg/service/growi_notice_sync.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"github.com/adrg/frontmatter"
+	"github.com/ictsc/growi_client"
+	"github.com/ictsc/ictsc-rikka/pkg/entity"
+	"github.com/ictsc/ictsc-rikka/pkg/repository"
+	"github.com/pkg/errors"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type GrowiNoticeSync struct {
+	client                   *growi_client.GrowiClient
+	path                     string
+	noticeWithInfoRepository repository.NoticeWithSyncTimeRepository
+	noticeRepository         repository.NoticeRepository
+}
+
+func NewGrowiNoticeSyncService(
+	client *growi_client.GrowiClient,
+	path string,
+	noticeWithInfoRepository repository.NoticeWithSyncTimeRepository,
+	noticeRepository repository.NoticeRepository,
+) *GrowiNoticeSync {
+	return &GrowiNoticeSync{
+		client:                   client,
+		path:                     path,
+		noticeWithInfoRepository: noticeWithInfoRepository,
+		noticeRepository:         noticeRepository,
+	}
+}
+
+func (s *GrowiNoticeSync) Sync(ctx context.Context) error {
+	notices, err := s.noticeRepository.GetAll()
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Failed to get notices").Error())
+	}
+
+	pages, err := s.client.GetSubordinatedPage(s.path)
+	if err != nil {
+		log.Fatalf(errors.Wrapf(err, "Failed to get subordinated list").Error())
+	}
+
+	// ProblemPath 以下のやつだけ同期
+	r := regexp.MustCompile(fmt.Sprintf(`^%s/`, s.path))
+
+	for _, page := range pages {
+		if r.MatchString(page.Path) {
+			// redis キャッシュから取得し
+			cachedNoticeWithInfo, err := s.noticeWithInfoRepository.Get(ctx, page.Path)
+			if err != nil {
+				fmt.Println("Not found in redis")
+			} else {
+				if cachedNoticeWithInfo.UpdatedAt == page.UpdatedAt {
+					fmt.Println("Not updated")
+					continue
+				}
+			}
+
+			noticePage, err := s.client.GetPage(page.Path)
+			if err != nil {
+				log.Fatal(errors.Wrap(err, "Failed to get notice page").Error())
+			}
+
+			var matter = &entity.NoticeFrontMatter{}
+
+			body, err := frontmatter.Parse(strings.NewReader(noticePage.Revision.Body), matter)
+			if err != nil {
+				fmt.Println(errors.Wrap(err, "Failed to parse frontmatter").Error())
+				continue
+			}
+			err = matter.Validate()
+			if err != nil {
+				fmt.Println(errors.Wrap(err, "Failed to validate frontmatter").Error())
+				continue
+			}
+
+			fmt.Println(matter)
+			fmt.Println(string(body))
+
+			split := strings.Split(page.Path, "/")
+			sourceId := split[len(split)-1]
+
+			// ここから更新処理
+			// 1. 最終日付を更新
+			// 2. notice をキャッシュ
+			newNoticeWithInfo := &entity.NoticeWithSyncTime{
+				Notice: entity.Notice{
+					Title:    matter.Title,
+					Body:     string(body),
+					SourceId: sourceId,
+				},
+				UpdatedAt: page.UpdatedAt,
+			}
+
+			var exists = false
+			for _, notice := range notices {
+				// path を スラッシュで区切って一番最後の文字列
+				// 例: /notice/2020/01/01/notice1 -> notice1
+				if notice.SourceId == sourceId {
+					newNoticeWithInfo.ID = notice.ID
+					newNoticeWithInfo.CreatedAt = notice.CreatedAt
+
+					exists = true
+					break
+
+				}
+			}
+
+			// 既に存在する場合は更新
+			if exists {
+				_, err = s.noticeRepository.Update(&newNoticeWithInfo.Notice)
+				if err != nil {
+					log.Println(err)
+					continue
+				}
+			} else {
+				newNoticeWithInfo.Notice.Base.CreatedAt = time.Now()
+
+				_, err = s.noticeRepository.Create(&newNoticeWithInfo.Notice)
+				if err != nil {
+					log.Println(err)
+					continue
+				}
+			}
+
+			err = s.noticeWithInfoRepository.Set(ctx, page.Path, *newNoticeWithInfo)
+			if err != nil {
+				log.Fatal(errors.Wrap(err, "Failed to set notice to redis").Error())
+			}
+		}
+	}
+
+	return nil
+}

pkg/service/growi_problem_sync.go

@@ -50,7 +50,7 @@ func (s *GrowiProblemSync) Sync(ctx context.Context) error {
 		log.Fatalf(errors.Wrapf(err, "Failed to get subordinated list").Error())
 	}
 
-	// Path 以下のやつだけ同期
+	// ProblemPath 以下のやつだけ同期
 	r := regexp.MustCompile(fmt.Sprintf(`^%s/`, s.path))
 
 	for _, page := range pages {
@@ -76,7 +76,7 @@ func (s *GrowiProblemSync) Sync(ctx context.Context) error {
 				log.Fatalf(errors.Wrapf(err, "Failed to get page").Error())
 			}
 
-			var matter = &entity.FrontMatter{}
+			var matter = &entity.ProblemFrontMatter{}
 
 			// frontmatter
 			// TODO(k-shir0): フォーマットもチェックする