Skip to content

Commit

Permalink
sealedev: Implement multithreading
Browse files Browse the repository at this point in the history
  • Loading branch information
@kodawah
kodawah committed Oct 14, 2023
1 parent f2ea41d commit 0238511
Showing 1 changed file with 137 additions and 55 deletions.
192 changes: 137 additions & 55 deletions sealedev/sealedev.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (
"fmt"
"io"
"strconv"
"sync"
"time"

"github.com/montanaflynn/stats"
"github.com/mtgban/go-mtgban/mtgban"
"github.com/mtgban/go-mtgban/mtgmatcher"
"github.com/mtgban/go-mtgban/tcgplayer"
"golang.org/x/exp/slices"
)

const (
Expand Down Expand Up @@ -105,6 +107,19 @@ func (ss *SealedEVScraper) printf(format string, a ...interface{}) {
}
}

type resultChan struct {
i int
ev float64
err error
}

type respChan struct {
productId string
invEntry *mtgban.InventoryEntry
buyEntry *mtgban.BuylistEntry
err error
}

func (ss *SealedEVScraper) scrape() error {
ss.printf("Loading BAN prices")
prices, err := loadPrices(ss.banpriceKey)
Expand All @@ -113,8 +128,13 @@ func (ss *SealedEVScraper) scrape() error {
}
ss.printf("Retrieved %d+%d prices", len(prices.Retail), len(prices.Buylist))

start := time.Now()

sets := mtgmatcher.GetSets()
for _, set := range sets {
var wgOut sync.WaitGroup
channelOut := make(chan respChan)

// Skip products without Sealed or Booster information
switch set.Code {
case "FBB", "4BB", "DRKITA", "LEGITA", "RIN", "4EDALT", "BCHR":
Expand All @@ -129,77 +149,139 @@ func (ss *SealedEVScraper) scrape() error {
}

for _, product := range set.SealedProduct {
var errPrinted bool

repeats := EVAverageRepetition
if ss.FastMode {
repeats = 10
}
if !mtgmatcher.SealedIsRandom(set.Code, product.UUID) {
repeats = 1
}
wgOut.Add(1)
go func() {
defer wgOut.Done()

datasets := make([][]float64, len(evParameters))

for j := 0; j < repeats; j++ {
picks, err := mtgmatcher.GetPicksForSealed(set.Code, product.UUID)
if err != nil {
if !errPrinted && product.Category != "land_station" {
ss.printf("[%s] '%s' error: %s", set.Code, product.Name, err.Error())
errPrinted = true
}
continue
repeats := EVAverageRepetition
if ss.FastMode {
repeats = 10
}

for i := range evParameters {
priceSource := prices.Retail
if evParameters[i].FoundInBuylist {
priceSource = prices.Buylist
}
ev := valueInBooster(picks, priceSource, evParameters[i].SourceName)
datasets[i] = append(datasets[i], ev)
if !mtgmatcher.SealedIsRandom(set.Code, product.UUID) {
repeats = 1
}
}

for i, dataset := range datasets {
price, err := evParameters[i].StatsFunc(dataset)
if err != nil {
continue
var wg sync.WaitGroup

datasets := make([][]float64, len(evParameters))
channel := make(chan resultChan)

for j := 0; j < repeats; j++ {
wg.Add(1)

go func() {
defer wg.Done()

picks, err := mtgmatcher.GetPicksForSealed(set.Code, product.UUID)
if err != nil {
if product.Category != "land_station" {
channel <- resultChan{
err: fmt.Errorf("[%s] '%s' error: %s", set.Code, product.Name, err.Error()),
}
}
return
}

for i := range evParameters {
priceSource := prices.Retail
if evParameters[i].FoundInBuylist {
priceSource = prices.Buylist
}
ev := valueInBooster(picks, priceSource, evParameters[i].SourceName)
channel <- resultChan{
i: i,
ev: ev,
}
}
}()
}

if price == 0 {
continue
go func() {
wg.Wait()
close(channel)
}()

for resp := range channel {
if resp.err != nil {
channelOut <- respChan{
err: resp.err,
}
continue
}

datasets[resp.i] = append(datasets[resp.i], resp.ev)
}

if evParameters[i].TargetsBuylist {
link := ckBuylistLink
if ss.BuylistAffiliate != "" {
link += fmt.Sprintf("?partner=%s&utm_campaign=%s&utm_medium=affiliate&utm_source=%s", ss.BuylistAffiliate, ss.BuylistAffiliate, ss.BuylistAffiliate)
for i, dataset := range datasets {
price, err := evParameters[i].StatsFunc(dataset)
if err != nil {
continue
}
ss.buylist.Add(product.UUID, &mtgban.BuylistEntry{
Conditions: "INDEX",
BuyPrice: price,
TradePrice: price * 1.3,
URL: link,
})
} else {
var link string
tcgID, _ := strconv.Atoi(product.Identifiers["tcgplayerProductId"])
if tcgID != 0 {
link = tcgplayer.TCGPlayerProductURL(tcgID, "", ss.Affiliate, "")

if price == 0 {
continue
}

ss.inventory.Add(product.UUID, &mtgban.InventoryEntry{
Conditions: "INDEX",
Price: price,
SellerName: evParameters[i].Name,
URL: link,
})
if evParameters[i].TargetsBuylist {
link := ckBuylistLink
if ss.BuylistAffiliate != "" {
link += fmt.Sprintf("?partner=%s&utm_campaign=%s&utm_medium=affiliate&utm_source=%s", ss.BuylistAffiliate, ss.BuylistAffiliate, ss.BuylistAffiliate)
}
channelOut <- respChan{
productId: product.UUID,
buyEntry: &mtgban.BuylistEntry{
Conditions: "INDEX",
BuyPrice: price,
TradePrice: price * 1.3,
URL: link,
},
}
} else {
var link string
tcgID, _ := strconv.Atoi(product.Identifiers["tcgplayerProductId"])
if tcgID != 0 {
link = tcgplayer.TCGPlayerProductURL(tcgID, "", ss.Affiliate, "")
}

channelOut <- respChan{
productId: product.UUID,
invEntry: &mtgban.InventoryEntry{
Conditions: "INDEX",
Price: price,
SellerName: evParameters[i].Name,
URL: link,
},
}
}
}
}()
}

go func() {
wgOut.Wait()
close(channelOut)
}()

var printedErrors []string
for result := range channelOut {
if result.err != nil && !slices.Contains(printedErrors, result.err.Error()) {
ss.printf("%s", result.err.Error())
printedErrors = append(printedErrors, result.err.Error())
continue
}

if result.invEntry != nil {
ss.inventory.Add(result.productId, result.invEntry)
}
if result.buyEntry != nil {
ss.buylist.Add(result.productId, result.buyEntry)
}
}

}

ss.printf("Took %v", time.Since(start))

ss.inventoryDate = time.Now()
ss.buylistDate = time.Now()

Expand Down

0 comments on commit 0238511

Please sign in to comment.