Restore miniaturemarket module for sealed

miniaturemarket/api.go

@@ -0,0 +1,100 @@
+package miniaturemarket
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+
+	http "github.com/hashicorp/go-retryablehttp"
+)
+
+type MMProduct struct {
+	UUID     string  `json:"uniqueId"`
+	Edition  string  `json:"mtg_set"`
+	Title    string  `json:"title"`
+	URL      string  `json:"productUrl"`
+	Price    float64 `json:"price"`
+	Quantity int     `json:"quantity"`
+}
+
+type MMSearchResponse struct {
+	Response struct {
+		NumberOfProducts int         `json:"numberOfProducts"`
+		Products         []MMProduct `json:"products"`
+	} `json:"response"`
+}
+
+type MMClient struct {
+	client *http.Client
+}
+
+const (
+	MMCategoryMtgSingles    = "1466"
+	MMDefaultResultsPerPage = 32
+
+	mmSearchURL = "https://search.unbxd.io/fb500edbf5c28edfa74cc90561fe33c3/prod-miniaturemarket-com811741582229555/category"
+)
+
+func NewMMClient() *MMClient {
+	mm := MMClient{}
+	mm.client = http.NewClient()
+	mm.client.Logger = nil
+	return &mm
+}
+
+func (mm *MMClient) NumberOfProducts() (int, error) {
+	resp, err := mm.query(0, 0)
+	if err != nil {
+		return 0, err
+	}
+	return resp.Response.NumberOfProducts, nil
+}
+
+func (mm *MMClient) GetInventory(start int) (*MMSearchResponse, error) {
+	return mm.query(start, MMDefaultResultsPerPage)
+}
+
+func (mm *MMClient) query(start, maxResults int) (*MMSearchResponse, error) {
+	u, err := url.Parse(mmSearchURL)
+	if err != nil {
+		return nil, err
+	}
+
+	q := u.Query()
+	q.Set("format", "json")
+	q.Set("version", "V2")
+	q.Set("start", fmt.Sprint(start))
+	q.Set("rows", fmt.Sprint(maxResults))
+	q.Set("variants", "true")
+	q.Set("variants.count", "10")
+	q.Set("fields", "*")
+	q.Set("facet.multiselect", "true")
+	q.Set("selectedfacet", "true")
+	q.Set("pagetype", "boolean")
+	q.Set("p", `categoryPath:"Trading Card Games"`)
+	q.Set("filter", `categoryPath1_fq:"Trading Card Games"`)
+	q.Set("filter", `categoryPath2_fq:"Trading Card Games>Magic the Gathering"`)
+	q.Set("filter", `stock_status_uFilter:"In Stock"`)
+	q.Set("filter", `manufacturer_uFilter:"Wizards of the Coast"`)
+	u.RawQuery = q.Encode()
+
+	resp, err := mm.client.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var search MMSearchResponse
+	err = json.Unmarshal(data, &search)
+	if err != nil {
+		return nil, err
+	}
+
+	return &search, nil
+}

miniaturemarket/miniaturemarket.go

@@ -0,0 +1,160 @@
+package miniaturemarket
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mtgban/go-mtgban/mtgban"
+)
+
+type Miniaturemarket struct {
+	LogCallback    mtgban.LogCallbackFunc
+	MaxConcurrency int
+	Affiliate      string
+
+	inventoryDate time.Time
+	client        *MMClient
+	inventory     mtgban.InventoryRecord
+}
+
+func NewScraperSealed() *Miniaturemarket {
+	mm := Miniaturemarket{}
+	mm.client = NewMMClient()
+	mm.inventory = mtgban.InventoryRecord{}
+	mm.MaxConcurrency = defaultConcurrency
+	return &mm
+}
+
+const (
+	defaultConcurrency = 6
+)
+
+type respChan struct {
+	cardId   string
+	invEntry *mtgban.InventoryEntry
+}
+
+func (mm *Miniaturemarket) printf(format string, a ...interface{}) {
+	if mm.LogCallback != nil {
+		mm.LogCallback("[MMSealed] "+format, a...)
+	}
+}
+
+func (mm *Miniaturemarket) processPage(channel chan<- respChan, start int) error {
+	resp, err := mm.client.GetInventory(start)
+	if err != nil {
+		return nil
+	}
+	resp = resp
+
+	for _, product := range resp.Response.Products {
+		productName := strings.TrimPrefix(product.Title, "Magic the Gathering: ")
+		productName = strings.TrimSuffix(productName, " (Preorder)")
+		edition := product.Edition
+
+		uuid, err := preprocessSealed(productName, edition)
+		if (err != nil || uuid == "") && strings.Contains(productName, "Commander") && !strings.Contains(edition, "Commander") {
+			uuid, err = preprocessSealed(productName, edition+" Commander")
+		}
+		if err != nil {
+			if err.Error() != "unsupported" {
+				mm.printf("%s in %s | %s", productName, edition, err.Error())
+			}
+			continue
+		}
+
+		if uuid == "" {
+			if !strings.Contains(productName, "Prerelease Pack") &&
+				!strings.Contains(productName, "Starter Kit") &&
+				!strings.Contains(productName, "Case") {
+				mm.printf("unable to parse %s in %s", productName, edition)
+			}
+			continue
+		}
+
+		link := product.URL
+		if mm.Affiliate != "" {
+			link += "?utm_source=" + mm.Affiliate + "&utm_medium=feed&utm_campaign=mtg_singles"
+		}
+
+		channel <- respChan{
+			cardId: uuid,
+			invEntry: &mtgban.InventoryEntry{
+				Price:    product.Price,
+				Quantity: product.Quantity,
+				URL:      link,
+			},
+		}
+	}
+
+	return nil
+}
+
+func (mm *Miniaturemarket) scrape() error {
+	pages := make(chan int)
+	channel := make(chan respChan)
+	var wg sync.WaitGroup
+
+	totalProducts, err := mm.client.NumberOfProducts()
+	if err != nil {
+		return err
+	}
+	mm.printf("Parsing %d items", totalProducts)
+
+	for i := 0; i < mm.MaxConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			for start := range pages {
+				err = mm.processPage(channel, start)
+				if err != nil {
+					mm.printf("%s", err.Error())
+				}
+			}
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		for i := 0; i < totalProducts; i += MMDefaultResultsPerPage {
+			pages <- i
+		}
+		close(pages)
+
+		wg.Wait()
+		close(channel)
+	}()
+
+	for record := range channel {
+		err := mm.inventory.Add(record.cardId, record.invEntry)
+		if err != nil {
+			mm.printf("%v", err)
+			continue
+		}
+	}
+
+	mm.inventoryDate = time.Now()
+
+	return nil
+}
+
+func (mm *Miniaturemarket) Inventory() (mtgban.InventoryRecord, error) {
+	if len(mm.inventory) > 0 {
+		return mm.inventory, nil
+	}
+
+	err := mm.scrape()
+	if err != nil {
+		return nil, err
+	}
+
+	return mm.inventory, nil
+}
+
+func (mm *Miniaturemarket) Info() (info mtgban.ScraperInfo) {
+	info.Name = "Miniature Market"
+	info.Shorthand = "MMSealed"
+	info.InventoryTimestamp = &mm.inventoryDate
+	info.SealedMode = true
+	return
+}

miniaturemarket/preprocess.go

@@ -0,0 +1,136 @@
+package miniaturemarket
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/mtgban/go-mtgban/mtgmatcher"
+)
+
+var sealedRenames = map[string]string{
+	"Adventures in the Forgotten Realms - Commander Deck Set": "Adventures in the Forgotten Realms Commander Deck Display",
+
+	"Kaldheim - Commander Deck Set (Set of 2)":           "Kaldheim Commander Decks - Set of 2",
+	"Kaladesh - Planeswalker Deck Set (2 Theme Decks)":   "Kaladesh Planeswalker Decks - Set of 2",
+	"Pioneer Challenger Deck Set (4)":                    "Pioneer Challenger Deck 2021 Set of 4",
+	"Pioneer Challenger Deck Set 2022 (4)":               "Pioneer Challenger Deck 2022 Set of 4",
+	"MtG Lost Caverns of Ixalan: Commander Deck Set (4)": "The Lost Caverns of Ixalan Commander Deck Case",
+
+	"2022 Starter Kit": "2022 Arena Starter Kit",
+}
+
+func preprocessSealed(productName, edition string) (string, error) {
+	switch {
+	case strings.Contains(productName, "Jumpstart 2022"):
+		edition = "Jumpstart 2022"
+	case strings.Contains(edition, "2022 Starter Kit"):
+		edition = "SNC"
+	}
+
+	// If edition is empty, do not return and instead loop through
+	var setCode string
+	set, err := mtgmatcher.GetSetByName(edition)
+	if err == nil {
+		setCode = set.Code
+	}
+
+	rename, found := sealedRenames[productName]
+	if found {
+		productName = rename
+	}
+
+	productName = strings.Replace(productName, "Deck Set (Set of 2)", "Decks - Set of 2", 1)
+	productName = strings.Replace(productName, "Deck Set (2)", "Decks - Set of 2", 1)
+	productName = strings.Replace(productName, "Deck Set (4)", "Decks - Set of 4", 1)
+	productName = strings.Replace(productName, "Deck Set (5)", "Decks - Set of 5", 1)
+	productName = strings.Replace(productName, "(Premium)", "Premium", 1)
+
+	if strings.Contains(edition, "Lost Caverns") {
+		edition = strings.Replace(edition, "MtG Lost Caverns", "The Lost Caverns", 1)
+		productName = strings.Replace(productName, "MtG Lost Caverns", "The Lost Caverns", 1)
+		if strings.Contains(productName, "Commander") {
+			edition = strings.TrimPrefix(edition, "The ")
+		}
+	}
+
+	edition = strings.Replace(edition, "Phyrexia - All Will Be One", "Phyrexia: All Will Be One", 1)
+	productName = strings.Replace(productName, "Phyrexia - All Will Be One", "Phyrexia: All Will Be One", 1)
+
+	if strings.Contains(edition, "Tales of Middle-earth") && !strings.Contains(edition, "The Lord of the Rings") {
+		edition = "The Lord of the Rings " + edition
+	}
+	if strings.Contains(productName, "Tales of Middle-earth") && !strings.Contains(productName, "The Lord of the Rings") {
+		productName = "The Lord of the Rings " + productName
+	}
+
+	productName = mtgmatcher.SplitVariants(productName)[0]
+
+	switch {
+	case strings.Contains(productName, "Land Station"),
+		strings.Contains(productName, "Variety Pack"),
+		strings.Contains(productName, "Scene Box"),
+		strings.Contains(productName, "Transformers TCG"):
+		return "", errors.New("unsupported")
+	}
+
+	var uuid string
+	for _, set := range mtgmatcher.GetSets() {
+		if setCode != "" && setCode != set.Code {
+			continue
+		}
+
+		for _, sealedProduct := range set.SealedProduct {
+			if mtgmatcher.SealedEquals(sealedProduct.Name, productName) {
+				uuid = sealedProduct.UUID
+				break
+			}
+		}
+
+		if uuid == "" {
+			for _, sealedProduct := range set.SealedProduct {
+				// If not found, look if the a chunk of the name is present in the deck name
+				switch {
+				case strings.Contains(productName, "Archenemy"),
+					strings.Contains(productName, "Duels of the Planeswalkers"),
+					strings.Contains(productName, "Commander"),
+					strings.Contains(productName, "Challenger Deck"),
+					strings.Contains(productName, "Secret Lair"),
+					strings.Contains(productName, "Planechase"):
+					decks, found := sealedProduct.Contents["deck"]
+					if found {
+						for _, deck := range decks {
+							// Work around internal names that are too long, like
+							// "Teeth of the Predator - the Garruk Wildspeaker Deck"
+							deckName := strings.Split(deck.Name, " - ")[0]
+							if mtgmatcher.SealedContains(productName, deckName) {
+								uuid = sealedProduct.UUID
+								break
+							}
+							// Scret Lair may have
+							deckName = strings.TrimSuffix(strings.ToLower(deckName), " foil")
+							if mtgmatcher.SealedContains(productName, deckName) {
+								uuid = sealedProduct.UUID
+								break
+							}
+						}
+					}
+				}
+				if uuid != "" {
+					break
+				}
+			}
+		}
+
+		// Last chance (in case edition is known)
+		if uuid == "" && setCode != "" && len(set.SealedProduct) == 1 {
+			uuid = set.SealedProduct[0].UUID
+		}
+
+		if uuid != "" {
+			break
+		}
+
+	}
+
+	return uuid, nil
+}