Skip to content

Commit

Permalink
Merge pull request #421 from penguin-statistics/dev
Browse files Browse the repository at this point in the history
  • Loading branch information
@GalvinGao
GalvinGao authored Mar 16, 2023
2 parents 2a48043 + 99536bc commit 72bca2b
Show file tree
Hide file tree
Showing 36 changed files with 1,513 additions and 918 deletions.
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0
go.opentelemetry.io/otel/sdk v1.14.0
go.uber.org/fx v1.19.1
go.uber.org/fx v1.19.2
golang.org/x/exp v0.0.0-20220823124025-807a23277127
golang.org/x/mod v0.7.0
golang.org/x/text v0.7.0
Expand Down Expand Up @@ -137,7 +137,7 @@ require (
go.opentelemetry.io/otel/metric v0.34.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/dig v1.16.0 // indirect
go.uber.org/dig v1.16.1 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,8 +539,12 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/dig v1.16.0 h1:O48QoUEj4ePocypAIE5jz+SrxVdG/izHM1CZ/Yjrwww=
go.uber.org/dig v1.16.0/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
go.uber.org/fx v1.19.1 h1:JwYIYAQzXBuBBwSZ1/tn/95pnQO/Sp3yE8lWj9eSAzI=
go.uber.org/fx v1.19.1/go.mod h1:bGK+AEy7XUwTBkqCsK/vDyFF0JJOA6X5KWpNC0e6qTA=
go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY=
go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
Expand Down
6 changes: 0 additions & 6 deletions internal/app/appconfig/spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,6 @@ type ConfigSpec struct {
// WorkerInterval describes the interval in-between different batches
WorkerInterval time.Duration `required:"true" split_words:"true" default:"10m"`

// WorkerTrendInterval describes the interval in-between different batches
WorkerTrendInterval time.Duration `required:"true" split_words:"true" default:"6h"`

// WorkerTrendEnabled describes whether to enable the trend worker
WorkerTrendEnabled bool `required:"true" split_words:"true" default:"true"`

// WorkerSeparation describes the separation time in-between different microtasks
WorkerSeparation time.Duration `required:"true" split_words:"true" default:"3s"`

Expand Down
94 changes: 60 additions & 34 deletions internal/controller/meta/admin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,22 +35,24 @@ import (
type AdminController struct {
fx.In

PatternRepo *repo.DropPattern
PatternElementRepo *repo.DropPatternElement
RecognitionDefectRepo *repo.RecognitionDefect
AdminService *service.Admin
ItemService *service.Item
StageService *service.Stage
DropMatrixService *service.DropMatrix
PatternMatrixService *service.PatternMatrix
TrendService *service.Trend
SiteStatsService *service.SiteStats
AnalyticsService *service.Analytics
UpyunService *service.Upyun
SnapshotService *service.Snapshot
DropReportService *service.DropReport
DropReportRepo *repo.DropReport
PropertyRepo *repo.Property
PatternRepo *repo.DropPattern
PatternElementRepo *repo.DropPatternElement
RecognitionDefectRepo *repo.RecognitionDefect
AdminService *service.Admin
ItemService *service.Item
StageService *service.Stage
DropMatrixService *service.DropMatrix
PatternMatrixService *service.PatternMatrix
TrendService *service.Trend
SiteStatsService *service.SiteStats
AnalyticsService *service.Analytics
UpyunService *service.Upyun
SnapshotService *service.Snapshot
DropReportService *service.DropReport
DropReportRepo *repo.DropReport
PropertyRepo *repo.Property
DropMatrixElementService *service.DropMatrixElement
TimeRangeService *service.TimeRange
}

func RegisterAdmin(admin *svr.Admin, c AdminController) {
Expand All @@ -68,9 +70,8 @@ func RegisterAdmin(admin *svr.Admin, c AdminController) {

admin.Get("/analytics/report-unique-users/by-source", c.GetRecentUniqueUserCountBySource)

admin.Get("/refresh/matrix/:server", c.RefreshAllDropMatrixElements)
admin.Get("/refresh/pattern/:server", c.RefreshAllPatternMatrixElements)
admin.Get("/refresh/trend/:server", c.RefreshAllTrendElements)
admin.Post("/refresh/matrix", c.CalcDropMatrixElements)
admin.Post("/refresh/pattern", c.CalcPatternMatrixElements)
admin.Get("/refresh/sitestats/:server", c.RefreshAllSiteStats)

admin.Get("/recognition/defects", c.GetRecognitionDefects)
Expand Down Expand Up @@ -322,21 +323,6 @@ func (c *AdminController) GetRecentUniqueUserCountBySource(ctx *fiber.Ctx) error
return ctx.JSON(result)
}

func (c *AdminController) RefreshAllDropMatrixElements(ctx *fiber.Ctx) error {
server := ctx.Params("server")
return c.DropMatrixService.RefreshAllDropMatrixElements(ctx.UserContext(), server, []string{constant.SourceCategoryAll})
}

func (c *AdminController) RefreshAllPatternMatrixElements(ctx *fiber.Ctx) error {
server := ctx.Params("server")
return c.PatternMatrixService.RefreshAllPatternMatrixElements(ctx.UserContext(), server, []string{constant.SourceCategoryAll})
}

func (c *AdminController) RefreshAllTrendElements(ctx *fiber.Ctx) error {
server := ctx.Params("server")
return c.TrendService.RefreshTrendElements(ctx.UserContext(), server, []string{constant.SourceCategoryAll})
}

func (c *AdminController) RefreshAllSiteStats(ctx *fiber.Ctx) error {
server := ctx.Params("server")
_, err := c.SiteStatsService.RefreshShimSiteStats(ctx.UserContext(), server)
Expand Down Expand Up @@ -569,3 +555,43 @@ func (c *AdminController) RecognitionItemsResourcesUpdated(ctx *fiber.Ctx) error

return ctx.JSON(asset)
}

func (c *AdminController) CalcDropMatrixElements(ctx *fiber.Ctx) error {
type calcDropMatrixElementsRequest struct {
Dates []string `json:"dates"`
Server string `json:"server"`
}
var request calcDropMatrixElementsRequest
if err := rekuest.ValidBody(ctx, &request); err != nil {
return err
}

for _, dateStr := range request.Dates {
date, err := time.Parse("2006-01-02", dateStr)
if err != nil {
return err
}
c.DropMatrixService.UpdateDropMatrixByGivenDate(ctx.UserContext(), request.Server, &date)
}
return ctx.SendStatus(fiber.StatusCreated)
}

func (c *AdminController) CalcPatternMatrixElements(ctx *fiber.Ctx) error {
type calcPatternMatrixElementsRequest struct {
Dates []string `json:"dates"`
Server string `json:"server"`
}
var request calcPatternMatrixElementsRequest
if err := rekuest.ValidBody(ctx, &request); err != nil {
return err
}

for _, dateStr := range request.Dates {
date, err := time.Parse("2006-01-02", dateStr)
if err != nil {
return err
}
c.PatternMatrixService.UpdatePatternMatrixByGivenDate(ctx.UserContext(), request.Server, &date)
}
return ctx.SendStatus(fiber.StatusCreated)
}
14 changes: 7 additions & 7 deletions internal/controller/v2/private.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package v2
import (
"time"

"exusiai.dev/gommon/constant"
"github.com/gofiber/fiber/v2"
"go.uber.org/fx"
"gopkg.in/guregu/null.v3"
Expand All @@ -13,7 +14,6 @@ import (
"exusiai.dev/backend-next/internal/pkg/middlewares"
"exusiai.dev/backend-next/internal/server/svr"
"exusiai.dev/backend-next/internal/service"
"exusiai.dev/gommon/constant"
)

var _ modelv2.Dummy
Expand Down Expand Up @@ -60,15 +60,15 @@ func (c *Private) GetDropMatrix(ctx *fiber.Ctx) error {
accountId.Valid = true
}

shimResult, err := c.DropMatrixService.GetShimMaxAccumulableDropMatrixResults(ctx.UserContext(), server, true, "", "", accountId, category)
shimResult, err := c.DropMatrixService.GetShimDropMatrix(ctx.UserContext(), server, true, "", "", accountId, category)
if err != nil {
return err
}

if !accountId.Valid {
key := server + constant.CacheSep + "true" + constant.CacheSep + category
var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimMaxAccumulableDropMatrixResults#server|showClosedZoned|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimGlobalDropMatrix#server|showClosedZoned|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand Down Expand Up @@ -101,15 +101,15 @@ func (c *Private) GetPatternMatrix(ctx *fiber.Ctx) error {
accountId.Valid = true
}

shimResult, err := c.PatternMatrixService.GetShimLatestPatternMatrixResults(ctx.UserContext(), server, accountId, category)
shimResult, err := c.PatternMatrixService.GetShimPatternMatrix(ctx.UserContext(), server, accountId, category)
if err != nil {
return err
}

if !accountId.Valid {
key := server + constant.CacheSep + category
var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimLatestPatternMatrixResults#server|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimGlobalPatternMatrix#server|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand All @@ -127,13 +127,13 @@ func (c *Private) GetPatternMatrix(ctx *fiber.Ctx) error {
// @Router /PenguinStats/api/v2/_private/result/trend/{server} [GET]
func (c *Private) GetTrends(ctx *fiber.Ctx) error {
server := ctx.Params("server")
shimResult, err := c.TrendService.GetShimSavedTrendResults(ctx.UserContext(), server)
shimResult, err := c.TrendService.GetShimTrend(ctx.UserContext(), server)
if err != nil {
return err
}

var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimSavedTrendResults#server:"+server+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimTrend#server:"+server+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand Down
17 changes: 6 additions & 11 deletions internal/controller/v2/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,6 @@ func (c *Result) GetDropMatrix(ctx *fiber.Ctx) error {
if err != nil {
return err
}
// if showClosedZones {
// flog.
// DebugFrom(ctx, "query.result.matrix.show_closed_zones").
// Msg("show_closed_zone is used")
// }
stageFilterStr := ctx.Query("stageFilter")
itemFilterStr := ctx.Query("itemFilter")

Expand All @@ -138,7 +133,7 @@ func (c *Result) GetDropMatrix(ctx *fiber.Ctx) error {
accountId.Valid = true
}

shimQueryResult, err := c.DropMatrixService.GetShimMaxAccumulableDropMatrixResults(ctx.UserContext(), server, showClosedZones, stageFilterStr, itemFilterStr, accountId, constant.SourceCategoryAll)
shimQueryResult, err := c.DropMatrixService.GetShimDropMatrix(ctx.UserContext(), server, showClosedZones, stageFilterStr, itemFilterStr, accountId, constant.SourceCategoryAll)
if err != nil {
return err
}
Expand All @@ -147,7 +142,7 @@ func (c *Result) GetDropMatrix(ctx *fiber.Ctx) error {
if useCache {
key := server + constant.CacheSep + strconv.FormatBool(showClosedZones) + constant.CacheSep + constant.SourceCategoryAll
var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimMaxAccumulableDropMatrixResults#server|showClosedZoned|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimGlobalDropMatrix#server|showClosedZoned|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand Down Expand Up @@ -186,15 +181,15 @@ func (c *Result) GetPatternMatrix(ctx *fiber.Ctx) error {
accountId.Valid = true
}

shimResult, err := c.PatternMatrixService.GetShimLatestPatternMatrixResults(ctx.UserContext(), server, accountId, constant.SourceCategoryAll)
shimResult, err := c.PatternMatrixService.GetShimPatternMatrix(ctx.UserContext(), server, accountId, constant.SourceCategoryAll)
if err != nil {
return err
}

if !accountId.Valid {
key := server + constant.CacheSep + constant.SourceCategoryAll
var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimLatestPatternMatrixResults#server|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimGlobalPatternMatrix#server|sourceCategory:"+key+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand All @@ -216,13 +211,13 @@ func (c *Result) GetTrends(ctx *fiber.Ctx) error {
return err
}

shimResult, err := c.TrendService.GetShimSavedTrendResults(ctx.UserContext(), server)
shimResult, err := c.TrendService.GetShimTrend(ctx.UserContext(), server)
if err != nil {
return err
}

var lastModifiedTime time.Time
if err := cache.LastModifiedTime.Get("[shimSavedTrendResults#server:"+server+"]", &lastModifiedTime); err != nil {
if err := cache.LastModifiedTime.Get("[shimTrend#server:"+server+"]", &lastModifiedTime); err != nil {
lastModifiedTime = time.Now()
}
cachectrl.OptIn(ctx, lastModifiedTime)
Expand Down
15 changes: 8 additions & 7 deletions internal/controller/v3/dataset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,13 @@ func (c Dataset) aggregateMatrix(ctx *fiber.Ctx) (*modelv2.DropMatrixQueryResult
accountId.Valid = true
}

queryResult, err := c.DropMatrixService.GetMaxAccumulableDropMatrixResults(ctx.UserContext(), server, "", ctx.Params("itemId"), accountId)
if err != nil {
return nil, err
}
// TODO: disable for now, because v3 is not ready yet
// queryResult, err := c.DropMatrixService.GetMaxAccumulableDropMatrixResults(ctx.UserContext(), server, "", ctx.Params("itemId"), accountId)
// if err != nil {
// return nil, err
// }

return queryResult, nil
return nil, nil
}

func (c Dataset) aggregateTrend(ctx *fiber.Ctx) (*modelv2.TrendQueryResult, error) {
Expand All @@ -65,7 +66,7 @@ func (c Dataset) aggregateTrend(ctx *fiber.Ctx) (*modelv2.TrendQueryResult, erro
return nil, err
}

result, err := c.TrendService.GetShimSavedTrendResults(ctx.UserContext(), server)
result, err := c.TrendService.GetShimTrend(ctx.UserContext(), server)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -94,7 +95,7 @@ func (c Dataset) aggregatePattern(ctx *fiber.Ctx) (*modelv3.PatternMatrixQueryRe
accountId.Valid = true
}

shimResult, err := c.PatternMatrixService.GetShimLatestPatternMatrixResults(ctx.UserContext(), server, accountId, constant.SourceCategoryAll)
shimResult, err := c.PatternMatrixService.GetShimPatternMatrix(ctx.UserContext(), server, accountId, constant.SourceCategoryAll)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit 72bca2b

Please sign in to comment.