Added additional poe.ninja price history data to requests; fixed poe.…

…ninja API calls (#1021) * fixed affix looup; added map poe ninja map prices to search results * ItemInfo data files * added poe.ninja price history data for some searches * updated poe.ninja API calls * added poe.ninja api url * fixed poe.ninja currency data urls * data file updates
PoE-TradeMacro · Sep 6, 2019 · 525c035 · 525c035
1 parent bb31d25
commit 525c035
Show file tree

Hide file tree

Showing 15 changed files with 422 additions and 248 deletions.
diff --git a/data/DivinationCardList.txt b/data/DivinationCardList.txt
diff --git a/data/GemQualityList.txt b/data/GemQualityList.txt
diff --git a/data/MapList.txt b/data/MapList.txt
diff --git a/data/Uniques.txt b/data/Uniques.txt
diff --git a/data_trade/currency_tags.json b/data_trade/currency_tags.json
diff --git a/data_trade/item_bases_armour.json b/data_trade/item_bases_armour.json
diff --git a/data_trade/mods.json b/data_trade/mods.json
diff --git a/data_trade/prophecy_details.json b/data_trade/prophecy_details.json
diff --git a/data_trade/uniques.json b/data_trade/uniques.json
diff --git a/resources/Updates_Trade.txt b/resources/Updates_Trade.txt
@@ -1,5 +1,12 @@
 The following is a list of what has been updated, starting with 1.1.0
 
+2.15.1
+================================================================================================
+* Fixed the url for the item affix lookup function.
+* Added poe.ninja price history data to normal price requests for certain item types (maps, currency, fossils).
+* Updated data files.
+* Fixed poe.ninja API urls, which where changed, breaking some requests.
+
 2.15.0
 ================================================================================================
 * Added support for the korean PoE game client.

diff --git a/resources/VersionTrade.txt b/resources/VersionTrade.txt
@@ -1,2 +1,2 @@
-TradeReleaseVersion := "v2.15.0"
+TradeReleaseVersion := "v2.15.1"
 TradeAHKVersionRequired := "1.1.26.00"
diff --git a/resources/ahk/POE-ItemInfo.ahk b/resources/ahk/POE-ItemInfo.ahk
@@ -11327,7 +11327,7 @@ LookUpAffixes() {
 				prefix	:= boots . chest . gloves . helmet . shield . gripType . ac . jw
 				StringLower, prefix, prefix
 
-				url		.= prefix "-" suffix ; ".html"
+				url		.= prefix "-" suffix ".html"
 			}
 			openWith := AssociatedProgram("html")
 			OpenWebPageWith(openWith, Url)
@@ -11995,6 +11995,46 @@ CurrencyDataDownloadURLtoJSON(url, sampleValue, critical = false, isFallbackRequ
 	Return parsedJSON
 }
 
+PoENinjaPriceDataDownloadURLtoJSON(url, category, critical = false, isFallbackRequest = false, league = "", project = "", tmpFileName = "", fallbackDir = "", ByRef usedFallback = false, CurlTimeout = 35) {	
+	errorMsg := "Parsing the " category " data (json) from poe.ninja failed.`n"
+	errorMsg .= "This should only happen when the servers are down / unavailable."
+	errorMsg .= "`n`n"
+	errorMsg .= "This can fix itself when the servers are up again and the data gets updated automatically or if you restart the script at such a time."
+	errorMsg .= "`n`n"
+	errorMsg .= "You can find a log file with some debug information:"
+	errorMsg .= "`n" """" A_ScriptDir "\temp\StartupLog.txt"""
+	errorMsg .= "`n`nTry opening the settings menu and selecting a league/making sure that one is selected."
+	errorMsg .= "`n`n"
+
+	errors := 0
+	parsingError := false	
+	Try {
+		options := ""
+		options	.= "`n" "TimeOut: " CurlTimeout
+
+		reqHeaders.push("Connection: keep-alive")
+		reqHeaders.push("Cache-Control: max-age=0")
+		reqHeaders.push("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")	
+		reqHeaders.push("User-Agent: Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
+		priceData := PoEScripts_Download(url, postData, reqHeaders, options, true, true, false, "", reqHeadersCurl)
+
+		deleteError := PoEScripts_SaveWriteTextFile(A_ScriptDir "\temp\" category "History_" league ".txt", priceData, "utf-8", true, true)
+
+		Try {
+			parsedJSON := JSON.Load(priceData)
+		} Catch e {
+			parsingError := true
+		}
+	} Catch error {
+		; first currency data parsing (script start)
+		If (critical and (not sampleValue or isFallbackRequest)) {
+			errors++
+		}
+	}
+
+	Return parsedJSON
+}
+
 FetchCurrencyData:
 	_CurrencyDataJSON	:= {}
 	currencyLeagues	:= ["Standard", "Hardcore", "tmpstandard", "tmphardcore", "eventstandard", "eventhardcore"]
@@ -12003,11 +12043,11 @@ FetchCurrencyData:
 	loggedTempLeagueCurrencyRequest := loggedTempLeagueCurrencyRequest ? loggedTempLeagueCurrencyRequest : false
 
 	Loop, % currencyLeagues.Length() {
-		currencyLeague := currencyLeagues[A_Index]
-		url  := "https://poe.ninja/api/Data/GetCurrencyOverview?league=" . currencyLeague
+		currencyLeague := currencyLeagues[A_Index]	
+		url	:= "https://poe.ninja/api/data/ItemOverview?league=" . currencyLeague . "&type=Currency"
 		file := A_ScriptDir . "\temp\currencyData_" . currencyLeague . ".json"
 
-		url		:= "https://poe.ninja/api/Data/GetCurrencyOverview?league=" . currencyLeague
+		url	:= "https://poe.ninja/api/data/ItemOverview?league=" . currencyLeague . "&type=Currency"
 		critical	:= StrLen(Globals.Get("LastCurrencyUpdate")) ? false : true
 		parsedJSON := CurrencyDataDownloadURLtoJSON(url, sampleValue, critical, false, currencyLeague, "PoE-ItemInfo", file, A_ScriptDir "\data", usedFallback, loggedCurrencyRequestAtStartup, loggedTempLeagueCurrencyRequest)		
 

diff --git a/resources/ahk/TradeMacro.ahk b/resources/ahk/TradeMacro.ahk
@@ -1270,6 +1270,15 @@ TradeFunc_Main(openSearchInBrowser = false, isAdvancedPriceCheck = false, isAdva
 			RequestParams.xbase := Item.BaseName
 			Item.UsedInSearch.type := Item.BaseName
 		}
+
+		Item.priceHistory := TradeFunc_FindMapHistoryData(Item.SubType, Item.MapTier)
+	}
+
+	/*
+		fossils
+		*/
+	If (Item.IsFossil) {
+		Item.priceHistory := TradeFunc_FindFossilHistoryData(Item.Name)
 	}
 
 	/*
@@ -1446,6 +1455,7 @@ TradeFunc_Main(openSearchInBrowser = false, isAdvancedPriceCheck = false, isAdva
 	ParsingError	:= ""
 	currencyUrl	:= ""	
 	If (Item.IsCurrency and not Item.IsEssence and TradeFunc_CurrencyFoundOnCurrencySearch(Item.Name)) {
+		Item.priceHistory := TradeFunc_FindCurrencyHistoryData(Item.Name)
 		If (!TradeOpts.AlternativeCurrencySearch or Item.IsFossil) {			
 			Html := TradeFunc_DoCurrencyRequest(Item.Name, openSearchInBrowser, 0, currencyUrl, error)
 			If (error) {
@@ -1559,6 +1569,36 @@ TradeFunc_Main(openSearchInBrowser = false, isAdvancedPriceCheck = false, isAdva
 	TradeGlobals.Set("AdvancedPriceCheckItem", {})
 }
 
+TradeFunc_FindMapHistoryData(baseType, tier) {
+	For key, value in MapHistoryData {
+		If (baseType = value.baseType and tier = value.mapTier) {
+			Return {"totalChange" : value.sparkline.totalChange, "chaosValue" : value.chaosValue, "exaltedValue" : value.exaltedValue}
+		}
+	}
+}
+TradeFunc_FindFossilHistoryData(name) {
+	For key, value in FossilHistoryData {
+		If (name = value.name) {
+			Return {"totalChange" : value.sparkline.totalChange, "chaosValue" : value.chaosValue, "exaltedValue" : value.exaltedValue}
+		}
+	}
+}
+TradeFunc_FindCurrencyHistoryData(name) {
+	For key, value in CurrencyHistoryData {
+		If (name = value.currencyTypeName) {			
+			obj := {}
+			If (value.receiveSparkLine.data.length) {
+				obj.totalChange := value.receiveSparkLine.totalChange
+			} Else {
+				obj.totalChange := value.lowConfidenceReceiveSparkLine.totalChange
+			}			
+			obj.chaosValue := value.chaosEquivalent
+
+			Return obj
+		}
+	}
+}
+
 TradeFunc_GetPoENinjaItemUrl(league, item) {	
 	url := "https://poe.ninja/"
 
@@ -2488,6 +2528,16 @@ TradeFunc_ParseCurrencyHtml(html, payload, ParsingError = "") {
 	Title .= "`n------------------------------ `n"
 	NoOfItemsToShow := TradeOpts.ShowItemResults
 
+	totalChangeSign := (Item.priceHistory.totalChange > 0) ? "+" : ""
+	If (Item.IsFossil or Item.IsCurrency) {
+		If (Item.priceHistory.exaltedValue >= 1) {
+			Title .= "poe.ninja price history: " Round(Item.priceHistory.exaltedValue, 2) " exalted."	
+		} Else {
+			Title .= "poe.ninja price history: " Round(Item.priceHistory.chaosValue, 2) " chaos."	
+		}
+		Title .= " Change: " totalChangeSign "" Round(Item.priceHistory.totalChange, 0) "% (last 7 days).`n`n"
+	}
+
 	Title .= StrPad("IGN" ,10)
 	Title .= StrPad("| Ratio",20)
 	Title .= "| " . StrPad("Buy  ",20, "Left")
@@ -2832,7 +2882,6 @@ TradeFunc_ParseHtmlToObj(html, payload, iLvl = "", ench = "", isItemAgeRequest =
 	NoOfItemsToShow := TradeOpts.ShowItemResults
 	results := []
 	accounts := {}
-
 	While A_Index < NoOfItemsToShow {
 		result := {}
 
@@ -3048,6 +3097,21 @@ TradeFunc_ParseHtml(html, payload, iLvl = "", ench = "", isItemAgeRequest = fals
 		Title .= "`n"
 	}
 
+	; add poe.ninja chaos equivalents
+	totalChangeSign := (Item.priceHistory.totalChange > 0) ? "+" : ""	
+	If (Item.IsMap) {
+		Title .= "poe.ninja price history: " Round(Item.priceHistory.chaosValue, 2) " chaos."		
+		Title .= " Change: " totalChangeSign "" Round(Item.priceHistory.totalChange, 0) "% (last 7 days).`n`n"
+	} 
+	Else If (Item.IsFossil) {
+		If (Item.priceHistory.exaltedValue >= 1) {
+			Title .= "poe.ninja price history: " Round(Item.priceHistory.exaltedValue, 2) " exalted."
+		} Else {
+			Title .= "poe.ninja price history: " Round(Item.priceHistory.chaosValue, 2) " chaos."	
+		}		
+		Title .= " Change: " totalChangeSign "" Round(Item.priceHistory.totalChange, 0) "% (last 7 days).`n`n"
+	}
+
 	NoOfItemsToShow := TradeOpts.ShowItemResults
 	; add table headers to tooltip
 	Title .= TradeFunc_ShowAcc(StrPad("Account",12), "|")
@@ -5701,6 +5765,10 @@ TradeSettingsUI_ChkCorruptedOverride:
 Return
 
 ReadPoeNinjaCurrencyData:
+	/*
+		https://poe.ninja/swagger/
+	*/
+
 	; Disable hotkey until currency data was parsed
 	key := TradeOpts.ChangeLeagueHotKey
 	loggedCurrencyRequestAtStartup := loggedCurrencyRequestAtStartup ? loggedCurrencyRequestAtStartup : false
@@ -5716,11 +5784,29 @@ ReadPoeNinjaCurrencyData:
 	isFallback	:= false
 	file			:= A_ScriptDir . "\temp\currencyData.json"
 	fallBackDir	:= A_ScriptDir . "\data_trade"
-	url			:= "https://poe.ninja/api/Data/GetCurrencyOverview?league=" . league
+	url			:= "https://poe.ninja/api/data/CurrencyOverview?league=" . league . "&type=Currency"
+
 	parsedJSON	:= CurrencyDataDownloadURLtoJSON(url, sampleValue, false, isFallback, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, loggedCurrencyRequestAtStartup, loggedTempLeagueCurrencyRequest, TradeOpts.CurlTimeout)
 
+	mapUrl		:= "https://poe.ninja/api/data/ItemOverview?league=" . league . "&type=Map"
+	parsedMapJSON	:= PoENinjaPriceDataDownloadURLtoJSON(mapUrl, "map", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+
+	fossilUrl		:= "https://poe.ninja/api/data/ItemOverview?league=" . league . "&type=Fossil"
+	parsedFossilJSON	:= PoENinjaPriceDataDownloadURLtoJSON(fossilUrl, "fossil", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+
+	/*
+	scarabUrl		:= "https://poe.ninja/api/data/itemoverview?=" . league . "&type=Scarab"
+	parsedScarabJSON	:= PoENinjaPriceDataDownloadURLtoJSON(scarabUrl, "scarab", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+	
+	essenceUrl		:= "https://poe.ninja/api/data/itemoverview?=" . league . "&type=Essence"
+	parsedEssenceJSON	:= PoENinjaPriceDataDownloadURLtoJSON(essenceUrl, "essence", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+	
+	fragmentUrl		:= "https://poe.ninja/api/data/itemoverview?=" . league . "&type=Fragment"
+	parsedFragmentJSON	:= PoENinjaPriceDataDownloadURLtoJSON(fragmentUrl, "fragment", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+	*/
+
 	; fallback to Standard and Hardcore league if used league seems to not be available
-	If (!parsedJSON.currencyDetails.length()) {
+	If (not parsedJSON.currencyDetails.length() or not parsedMapJSON.lines.length()) {
 		isFallback	:= true
 		If (InStr(league, "Hardcore", 0) or RegExMatch(league, "HC")) {
 			league	:= "Hardcore"
@@ -5729,11 +5815,23 @@ ReadPoeNinjaCurrencyData:
 			league	:= "Standard"
 			fallback	:= "Standard"
 		}
-
-		url			:= "https://poe.ninja/api/Data/GetCurrencyOverview?league=" . league
-		parsedJSON	:= CurrencyDataDownloadURLtoJSON(url, sampleValue, true, isFallback, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, loggedCurrencyRequestAtStartup, loggedTempLeagueCurrencyRequest, TradeOpts.CurlTimeout)
+
+		If (not parsedJSON.currencyDetails.length()) {
+			url			:= "https://poe.ninja/api/data/CurrencyOverview?league=" . league . "&type=Currency"
+			parsedJSON	:= CurrencyDataDownloadURLtoJSON(url, sampleValue, true, isFallback, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, loggedCurrencyRequestAtStartup, loggedTempLeagueCurrencyRequest, TradeOpts.CurlTimeout)	
+		}
+		If (not parsedMapJSON.lines.length()) {
+			mapUrl		:= "https://poe.ninja/api/data/ItemOverview?league=" . league . "&type=Map"
+			parsedMapJSON	:= PoENinjaPriceDataDownloadURLtoJSON(mapUrl, "map", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+		}
+		If (not parsedFossilJSON.lines.length()) {
+			fossilUrl		:= "https://poe.ninja/api/data/ItemOverview?league=" . league . "&type=Fossil"
+			parsedFossilJSON	:= PoENinjaPriceDataDownloadURLtoJSON(fossilUrl, "fossil", true, false, league, "PoE-TradeMacro", file, fallBackDir, usedFallback, TradeOpts.CurlTimeout)
+		}
 	}
 	global CurrencyHistoryData := parsedJSON.lines
+	global MapHistoryData := parsedMapJSON.lines
+	global FossilHistoryData := parsedFossilJSON.lines
 	TradeGlobals.Set("LastAltCurrencyUpdate", A_NowUTC)
 
 	global ChaosEquivalents	:= {}
@@ -5742,7 +5840,7 @@ ReadPoeNinjaCurrencyData:
 		ChaosEquivalents[currencyBaseName] := val.chaosEquivalent
 	}
 	ChaosEquivalents["Chaos Orb"] := 1
-	
+
 	If (TempChangingLeagueInProgress) {
 		msg := "Changing league to " . TradeOpts.SearchLeague " (" . TradeGlobals.Get("LeagueName") . ") finished."
 		msg .= "`n- Requested chaos equivalents and currency history from poe.ninja."

diff --git a/resources/ahk/TradeMacroInit.ahk b/resources/ahk/TradeMacroInit.ahk
@@ -250,7 +250,10 @@ WriteTradeConfig(TradeConfigDir = "", TradeConfigFile = "config_trade.ini") {
 	oldLeague := TradeOpts.SearchLeague
 	oldAltCurrencySearch := TradeOpts.AlternativeCurrencySearch
 
-	UpdateOldTradeOptsFromVars()
+	UpdateOldTradeOptsFromVars()	
+	TradeOpts.ShowItemResults := RegExReplace(TradeOpts.ShowItemResults, "\D")
+	TradeOpts.ShowItemResults := (not StrLen(TradeOpts.ShowItemResults)) ? 15 : TradeOpts.ShowItemResults
+
 	TradeOpts.SearchLeague := TradeFunc_CheckIfLeagueIsActive(TradeOpts.SearchLeague, "2")
 	oldLeagueName := TradeGlobals.Get("LeagueName")
 	newLeagueName := TradeGlobals.Get("Leagues")[TradeOpts.SearchLeague]
@@ -264,6 +267,7 @@ WriteTradeConfig(TradeConfigDir = "", TradeConfigFile = "config_trade.ini") {
 	If ((TradeOpts.SearchLeague != oldLeague and AlternativeCurrencySearch) or (AlternativeCurrencySearch and oldAltCurrencySearch != AlternativeCurrencySearch)) {
 		GoSub, ReadPoeNinjaCurrencyData
 	}
+
 	TradeOpts_New := UpdateNewTradeOptsFromOld(TradeOpts_New)
 	TradeFunc_SyncUpdateSettings()
 	TradeFunc_AssignAllHotkeys()

diff --git a/resources/ahk/jsonData.ahk b/resources/ahk/jsonData.ahk
@@ -54,8 +54,11 @@ If (PoEScripts_SaveWriteTextFile(A_ScriptDir "\temp\currentLeagues.json", parsed
 }
 
 errorMsg := "Parsing the league data (json) from the Path of Exile API failed."
-errorMsg .= "`nThis should only happen when the servers are down for maintenance." 
-errorMsg .= "`n`nThe script execution will be stopped, please try again at a later time."
+errorMsg .= "`nThis should only happen when the pathofexile.com servers are down for maintenance or if you have network issues and can't connect to the site." 
+errorMsg .= "`n`nMost likely the servers are down, please check by visiting pathofexile.com and try again later."
+errorMsg .= "`n`nThe script execution will be stopped."
+errorMsg .= "`n`nPlease do not report this issue if the pathofexile.com servers are down for maintenance!"
+errorMsg .= "`nNobody can help you in that case, you'll have to wait until they are up again."
 
 Try {	
 	test := FileExist(A_ScriptDir "\temp\currentLeagues.json")