From 3d2d922a19051ec6185c95a563273d54ab3a8e8e Mon Sep 17 00:00:00 2001 From: K-shir0 <50326556+K-shir0@users.noreply.github.com> Date: Thu, 2 Mar 2023 16:44:26 +0900 Subject: [PATCH] =?UTF-8?q?update:=20=E9=80=9A=E7=9F=A5=E5=90=8C=E6=9C=9F?= =?UTF-8?q?=E3=82=B7=E3=82=B9=E3=83=86=E3=83=A0=E3=82=92=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=20(#121)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/rikka/main.go | 1 + cmd/ritto/.gitignore | 3 +- cmd/ritto/config.go | 11 +- cmd/ritto/main.go | 12 +- pkg/entity/notice.go | 19 +++ pkg/entity/notice_front_matter.go | 16 ++ ...ront_matter.go => problem_front_matter.go} | 2 +- pkg/repository/mariadb/notice_repository.go | 45 ++++++ pkg/repository/notice_repository.go | 13 ++ pkg/repository/notice_with_sync_time.go | 11 ++ pkg/repository/rc/notice_with_sync_time.go | 38 +++++ pkg/service/growi_notice_sync.go | 140 ++++++++++++++++++ pkg/service/growi_problem_sync.go | 4 +- 13 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 pkg/entity/notice.go create mode 100644 pkg/entity/notice_front_matter.go rename pkg/entity/{front_matter.go => problem_front_matter.go} (89%) create mode 100644 pkg/repository/mariadb/notice_repository.go create mode 100644 pkg/repository/notice_repository.go create mode 100644 pkg/repository/notice_with_sync_time.go create mode 100644 pkg/repository/rc/notice_with_sync_time.go create mode 100644 pkg/service/growi_notice_sync.go diff --git a/cmd/rikka/main.go b/cmd/rikka/main.go index 9357b79..80163f0 100644 --- a/cmd/rikka/main.go +++ b/cmd/rikka/main.go @@ -74,6 +74,7 @@ func init() { &entity.Problem{}, &entity.Answer{}, &entity.Attachment{}, + &entity.Notice{}, ) store, err = redis.NewStore( diff --git a/cmd/ritto/.gitignore b/cmd/ritto/.gitignore index a539470..26ca1c3 100644 --- a/cmd/ritto/.gitignore +++ b/cmd/ritto/.gitignore @@ -1 +1,2 @@ -config.yaml \ No newline at end of file +config.yaml +config.yaml.prod \ No newline at end of file diff --git a/cmd/ritto/config.go b/cmd/ritto/config.go index 5845898..cc0b0e8 100644 --- a/cmd/ritto/config.go +++ b/cmd/ritto/config.go @@ -30,11 +30,12 @@ type Rikka struct { } type GrowiConfig struct { - Url string `yaml:"url"` - Token string `yaml:"token"` - Path string `yaml:"path"` - Username string `yaml:"username"` - Password string `yaml:"password"` + Url string `yaml:"url"` + Token string `yaml:"token"` + ProblemPath string `yaml:"problemPath"` + NoticePath string `yaml:"noticePath"` + Username string `yaml:"username"` + Password string `yaml:"password"` } func (c *MariaDBConfig) getDSN() string { diff --git a/cmd/ritto/main.go b/cmd/ritto/main.go index 4e538e0..f653da0 100644 --- a/cmd/ritto/main.go +++ b/cmd/ritto/main.go @@ -86,6 +86,8 @@ func main() { growiSessionCookieRepo := rc.NewGrowiSessionCookieRepository(redisClient) problemWithSyncTimeRepo := rc.NewProblemWithSyncTimeRepository(redisClient) problemRepo := mariadb.NewProblemRepository(db) + noticeRepo := mariadb.NewNoticeRepository(db) + noticeWithSyncTimeRepo := rc.NewNoticeWithSyncTimeRepository(redisClient) option := &growi_client.GrowiClientOption{ Username: config.Growi.Username, @@ -115,16 +117,24 @@ func main() { ) growiProblemSyncService := service.NewGrowiProblemSyncService( client, - config.Growi.Path, + config.Growi.ProblemPath, config.Rikka.AuthorId, problemWithSyncTimeRepo, problemRepo, ) + growiNoticeSyncService := service.NewGrowiNoticeSyncService( + client, + config.Growi.NoticePath, + noticeWithSyncTimeRepo, + noticeRepo, + ) err = growiClientInitService.Init(ctx) // TODO(k-shir0): エラー処理追加 err = growiProblemSyncService.Sync(ctx) // TODO(k-shir0): エラー処理追加 + err = growiNoticeSyncService.Sync(ctx) + // TODO(k-shir0): エラー処理追加 if err != nil { log.Fatal(err) } diff --git a/pkg/entity/notice.go b/pkg/entity/notice.go new file mode 100644 index 0000000..e542f3f --- /dev/null +++ b/pkg/entity/notice.go @@ -0,0 +1,19 @@ +package entity + +import ( + "time" +) + +type Notice struct { + Base + + SourceId string `json:"source_id"` + Title string `json:"title"` + Body string `json:"body,omitempty"` + Draft bool `json:"draft"` +} + +type NoticeWithSyncTime struct { + Notice + UpdatedAt time.Time +} diff --git a/pkg/entity/notice_front_matter.go b/pkg/entity/notice_front_matter.go new file mode 100644 index 0000000..8688c2f --- /dev/null +++ b/pkg/entity/notice_front_matter.go @@ -0,0 +1,16 @@ +package entity + +import "errors" + +type NoticeFrontMatter struct { + Title string `yaml:"title"` + Draft bool `yaml:"draft"` +} + +func (n *NoticeFrontMatter) Validate() error { + if n.Title == "" { + return errors.New("title must not be empty") + } + + return nil +} diff --git a/pkg/entity/front_matter.go b/pkg/entity/problem_front_matter.go similarity index 89% rename from pkg/entity/front_matter.go rename to pkg/entity/problem_front_matter.go index 4cd87be..7cbee03 100644 --- a/pkg/entity/front_matter.go +++ b/pkg/entity/problem_front_matter.go @@ -1,6 +1,6 @@ package entity -type FrontMatter struct { +type ProblemFrontMatter struct { Code string `yaml:"code"` Title string `yaml:"title"` Point uint `yaml:"point"` diff --git a/pkg/repository/mariadb/notice_repository.go b/pkg/repository/mariadb/notice_repository.go new file mode 100644 index 0000000..25a52bd --- /dev/null +++ b/pkg/repository/mariadb/notice_repository.go @@ -0,0 +1,45 @@ +package mariadb + +import ( + "github.com/google/uuid" + "github.com/ictsc/ictsc-rikka/pkg/entity" + "gorm.io/gorm" +) + +type NoticeRepository struct { + db *gorm.DB +} + +func NewNoticeRepository(db *gorm.DB) *NoticeRepository { + return &NoticeRepository{ + db: db, + } +} + +func (r *NoticeRepository) Create(notice *entity.Notice) (*entity.Notice, error) { + err := r.db.Create(notice).Error + if err != nil { + return nil, err + } + return r.FindByID(notice.ID) +} + +func (r *NoticeRepository) GetAll() ([]*entity.Notice, error) { + notice := make([]*entity.Notice, 0) + err := r.db.Find(¬ice).Error + return notice, err +} + +func (r *NoticeRepository) FindByID(id uuid.UUID) (*entity.Notice, error) { + res := &entity.Notice{} + err := r.db.First(res, id).Error + return res, err +} + +func (r *NoticeRepository) Update(notice *entity.Notice) (*entity.Notice, error) { + err := r.db.Save(notice).Error + if err != nil { + return nil, err + } + return r.FindByID(notice.ID) +} diff --git a/pkg/repository/notice_repository.go b/pkg/repository/notice_repository.go new file mode 100644 index 0000000..58c5e7a --- /dev/null +++ b/pkg/repository/notice_repository.go @@ -0,0 +1,13 @@ +package repository + +import ( + "github.com/google/uuid" + "github.com/ictsc/ictsc-rikka/pkg/entity" +) + +type NoticeRepository interface { + Create(notice *entity.Notice) (*entity.Notice, error) + GetAll() ([]*entity.Notice, error) + FindByID(id uuid.UUID) (*entity.Notice, error) + Update(notice *entity.Notice) (*entity.Notice, error) +} diff --git a/pkg/repository/notice_with_sync_time.go b/pkg/repository/notice_with_sync_time.go new file mode 100644 index 0000000..6af52e3 --- /dev/null +++ b/pkg/repository/notice_with_sync_time.go @@ -0,0 +1,11 @@ +package repository + +import ( + "context" + "github.com/ictsc/ictsc-rikka/pkg/entity" +) + +type NoticeWithSyncTimeRepository interface { + Set(context.Context, string, entity.NoticeWithSyncTime) error + Get(context.Context, string) (*entity.NoticeWithSyncTime, error) +} diff --git a/pkg/repository/rc/notice_with_sync_time.go b/pkg/repository/rc/notice_with_sync_time.go new file mode 100644 index 0000000..05f8564 --- /dev/null +++ b/pkg/repository/rc/notice_with_sync_time.go @@ -0,0 +1,38 @@ +package rc + +import ( + "context" + "encoding/json" + "github.com/go-redis/redis/v8" + "github.com/ictsc/ictsc-rikka/pkg/entity" +) + +type NoticeWithSyncTimeRepository struct { + rc *redis.Client +} + +func NewNoticeWithSyncTimeRepository(rc *redis.Client) *NoticeWithSyncTimeRepository { + return &NoticeWithSyncTimeRepository{rc: rc} +} + +func (r *NoticeWithSyncTimeRepository) Set(ctx context.Context, path string, noticeWithInfo entity.NoticeWithSyncTime) error { + jsonBytes, err := json.Marshal(noticeWithInfo) + if err != nil { + return err + } + + err = r.rc.Set(ctx, path, jsonBytes, 0).Err() + return err +} + +func (r *NoticeWithSyncTimeRepository) Get(ctx context.Context, path string) (*entity.NoticeWithSyncTime, error) { + noticeWithInfo, err := r.rc.Get(ctx, path).Result() + if err != nil { + return nil, err + } + + noticeWithInfoEntity := entity.NoticeWithSyncTime{} + err = json.Unmarshal([]byte(noticeWithInfo), ¬iceWithInfoEntity) + + return ¬iceWithInfoEntity, err +} diff --git a/pkg/service/growi_notice_sync.go b/pkg/service/growi_notice_sync.go new file mode 100644 index 0000000..908ef1f --- /dev/null +++ b/pkg/service/growi_notice_sync.go @@ -0,0 +1,140 @@ +package service + +import ( + "context" + "fmt" + "github.com/adrg/frontmatter" + "github.com/ictsc/growi_client" + "github.com/ictsc/ictsc-rikka/pkg/entity" + "github.com/ictsc/ictsc-rikka/pkg/repository" + "github.com/pkg/errors" + "log" + "regexp" + "strings" + "time" +) + +type GrowiNoticeSync struct { + client *growi_client.GrowiClient + path string + noticeWithInfoRepository repository.NoticeWithSyncTimeRepository + noticeRepository repository.NoticeRepository +} + +func NewGrowiNoticeSyncService( + client *growi_client.GrowiClient, + path string, + noticeWithInfoRepository repository.NoticeWithSyncTimeRepository, + noticeRepository repository.NoticeRepository, +) *GrowiNoticeSync { + return &GrowiNoticeSync{ + client: client, + path: path, + noticeWithInfoRepository: noticeWithInfoRepository, + noticeRepository: noticeRepository, + } +} + +func (s *GrowiNoticeSync) Sync(ctx context.Context) error { + notices, err := s.noticeRepository.GetAll() + if err != nil { + log.Fatal(errors.Wrap(err, "Failed to get notices").Error()) + } + + pages, err := s.client.GetSubordinatedPage(s.path) + if err != nil { + log.Fatalf(errors.Wrapf(err, "Failed to get subordinated list").Error()) + } + + // ProblemPath 以下のやつだけ同期 + r := regexp.MustCompile(fmt.Sprintf(`^%s/`, s.path)) + + for _, page := range pages { + if r.MatchString(page.Path) { + // redis キャッシュから取得し + cachedNoticeWithInfo, err := s.noticeWithInfoRepository.Get(ctx, page.Path) + if err != nil { + fmt.Println("Not found in redis") + } else { + if cachedNoticeWithInfo.UpdatedAt == page.UpdatedAt { + fmt.Println("Not updated") + continue + } + } + + noticePage, err := s.client.GetPage(page.Path) + if err != nil { + log.Fatal(errors.Wrap(err, "Failed to get notice page").Error()) + } + + var matter = &entity.NoticeFrontMatter{} + + body, err := frontmatter.Parse(strings.NewReader(noticePage.Revision.Body), matter) + if err != nil { + fmt.Println(errors.Wrap(err, "Failed to parse frontmatter").Error()) + continue + } + err = matter.Validate() + if err != nil { + fmt.Println(errors.Wrap(err, "Failed to validate frontmatter").Error()) + continue + } + + fmt.Println(matter) + fmt.Println(string(body)) + + split := strings.Split(page.Path, "/") + sourceId := split[len(split)-1] + + // ここから更新処理 + // 1. 最終日付を更新 + // 2. notice をキャッシュ + newNoticeWithInfo := &entity.NoticeWithSyncTime{ + Notice: entity.Notice{ + Title: matter.Title, + Body: string(body), + SourceId: sourceId, + }, + UpdatedAt: page.UpdatedAt, + } + + var exists = false + for _, notice := range notices { + // path を スラッシュで区切って一番最後の文字列 + // 例: /notice/2020/01/01/notice1 -> notice1 + if notice.SourceId == sourceId { + newNoticeWithInfo.ID = notice.ID + newNoticeWithInfo.CreatedAt = notice.CreatedAt + + exists = true + break + + } + } + + // 既に存在する場合は更新 + if exists { + _, err = s.noticeRepository.Update(&newNoticeWithInfo.Notice) + if err != nil { + log.Println(err) + continue + } + } else { + newNoticeWithInfo.Notice.Base.CreatedAt = time.Now() + + _, err = s.noticeRepository.Create(&newNoticeWithInfo.Notice) + if err != nil { + log.Println(err) + continue + } + } + + err = s.noticeWithInfoRepository.Set(ctx, page.Path, *newNoticeWithInfo) + if err != nil { + log.Fatal(errors.Wrap(err, "Failed to set notice to redis").Error()) + } + } + } + + return nil +} diff --git a/pkg/service/growi_problem_sync.go b/pkg/service/growi_problem_sync.go index d3f72d0..e4d1289 100644 --- a/pkg/service/growi_problem_sync.go +++ b/pkg/service/growi_problem_sync.go @@ -50,7 +50,7 @@ func (s *GrowiProblemSync) Sync(ctx context.Context) error { log.Fatalf(errors.Wrapf(err, "Failed to get subordinated list").Error()) } - // Path 以下のやつだけ同期 + // ProblemPath 以下のやつだけ同期 r := regexp.MustCompile(fmt.Sprintf(`^%s/`, s.path)) for _, page := range pages { @@ -76,7 +76,7 @@ func (s *GrowiProblemSync) Sync(ctx context.Context) error { log.Fatalf(errors.Wrapf(err, "Failed to get page").Error()) } - var matter = &entity.FrontMatter{} + var matter = &entity.ProblemFrontMatter{} // frontmatter // TODO(k-shir0): フォーマットもチェックする