Merge pull request #464 from 0xProject/v5.1.0-beta

Version 5.1.0-beta

Author: fabioberger

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
       - image: circleci/golang:1.12.9-browsers
       - image: 0xorg/ganache-cli:latest
         environment:
-            VERSION: 4.3.0
+            VERSION: 4.3.3
     working_directory: /go/src/github.com/0xProject/0x-mesh
     steps:
       - checkout

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 This changelog is a work in progress and may contain notes for versions which have not actually been released. Check the [Releases](https://github.com/0xProject/0x-mesh/releases) page to see full release notes and more information about the latest released versions.
 
+## v5.1.0-beta
+
+### Features ✅ 
+
+- The `getStats` RPC endpoint now includes a new field which accounts for the number of orders that have been marked as "removed" but not yet permanently deleted ([#461](https://github.com/0xProject/0x-mesh/pull/461)).
+- Improved historical order sharing using round-robin algorithm instead of random selection ([#454](https://github.com/0xProject/0x-mesh/pull/454)). This will reduce the warm-up time for receiving existing orders when first joining the network.
+- Added ERC1155 assetData support ([#453](https://github.com/0xProject/0x-mesh/pull/453)). This includes order watching and order events for orders involving ERC1155 tokens.
+- Added Ability to specify custom contract addresses via the `CUSTOM_ADDRESSES` environment variable or the `customAddresses` field in the TypeScript bindings ([#451](https://github.com/0xProject/0x-mesh/pull/451)).
+
+### Bug fixes 🐞 
+
+- Temporarily disabled bandwidth-based peer banning ([#448](https://github.com/0xProject/0x-mesh/pull/448)). A [bug in libp2p](https://github.com/libp2p/go-libp2p-core/issues/65) was occasionally causing observed bandwidth usage to spike to unrealistic levels, which can result in peers being erroneously banned. We decided to temporarily stop banning peers while we're working with the libp2p team to resolve the issue.
+
 ## v5.0.0-beta
 
 ### Breaking changes 🛠 

README.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-5.0.0--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-5.1.0--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 [![Docs](https://img.shields.io/badge/docs-website-yellow.svg)](https://0x-org.gitbook.io/mesh)
 [![Chat with us on Discord](https://img.shields.io/badge/chat-Discord-blueViolet.svg)](https://discord.gg/HF7fHwk)
 [![Circle CI](https://img.shields.io/circleci/project/0xProject/0x-mesh/master.svg)](https://circleci.com/gh/0xProject/0x-mesh/tree/master)

browser/go/main.go

@@ -120,6 +120,9 @@ func convertConfig(jsConfig js.Value) (core.Config, error) {
 	if ethereumRPCMaxContentLength := jsConfig.Get("ethereumRPCMaxContentLength"); !isNullOrUndefined(ethereumRPCMaxContentLength) {
 		config.EthereumRPCMaxContentLength = ethereumRPCMaxContentLength.Int()
 	}
+	if customContractAddresses := jsConfig.Get("customContractAddresses"); !isNullOrUndefined(customContractAddresses) {
+		config.CustomContractAddresses = customContractAddresses.String()
+	}
 
 	return config, nil
 }

browser/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@0x/mesh-browser",
-    "version": "5.0.0-beta",
+    "version": "5.1.0-beta",
     "description": "TypeScript and JavaScript bindings for running Mesh directly in the browser.",
     "main": "./lib/index.js",
     "license": "Apache-2.0",

browser/ts/index.ts

@@ -89,6 +89,11 @@ func updateHardCodedVersions(version string) {
 	regex = `version(.*)= "(.*)"`
 	updateFileWithRegex(corePath, regex, newVersionString)
 
+	// Update `docs/deployment_with_telemetry.md`
+	newVersionString = fmt.Sprintf(`image: 0xorg/mesh:%s`, version)
+	regex = `image: 0xorg/mesh:latest`
+	updateFileWithRegex("docs/deployment_with_telemetry.md", regex, newVersionString)
+
 	// Update `CHANGELOG.md`
 	changelog := "CHANGELOG.md"
 	newChangelogSection := fmt.Sprintf(`## v%s`, version)

cmd/cut-release/main.go

@@ -52,6 +52,9 @@ var GanacheAccountToPrivateKey = map[common.Address][]byte{
 // GanacheDummyERC721TokenAddress is the dummy ERC721 token address in the Ganache snapshot
 var GanacheDummyERC721TokenAddress = common.HexToAddress("0x07f96aa816c1f244cbc6ef114bb2b023ba54a2eb")
 
+// GanacheDummyERC1155MintableAddress is the dummy ERC1155 token address in the Ganache snapshot
+var GanacheDummyERC1155MintableAddress = common.HexToAddress("0xc4df27466183c0fe2a5924d6ea56e334deff146a")
+
 // ErrInternal is used whenever we don't wish to expose internal errors to a client
 var ErrInternal = errors.New("internal error")
 

constants/constants.go

@@ -48,7 +48,7 @@ const (
 	expirationPollingInterval  = 50 * time.Millisecond
 	// logStatsInterval is how often to log stats for this node.
 	logStatsInterval = 5 * time.Minute
-	version          = "5.0.0-beta"
+	version          = "5.1.0-beta"
 )
 
 // Note(albrow): The Config type is currently copied to browser/ts/index.ts. We
@@ -97,6 +97,23 @@ type Config struct {
 	// or Infura. If using Alchemy or Parity, feel free to double the default max in order to reduce the
 	// number of RPC calls made by Mesh.
 	EthereumRPCMaxContentLength int `envvar:"ETHEREUM_RPC_MAX_CONTENT_LENGTH" default:"524288"`
+	// CustomContractAddresses is a JSON-encoded string representing a set of
+	// custom addresses to use for the configured network ID. The contract
+	// addresses for most common networks are already included by default, so this
+	// is typically only needed for testing on custom networks. The given
+	// addresses are added to the default list of addresses for known networks and
+	// overriding any contract addresses for known networks is not allowed. The
+	// addresses for exchange, devUtils, erc20Proxy, and erc721Proxy are required
+	// for each network. For example:
+	//
+	//    {
+	//        "exchange":"0x48bacb9266a570d521063ef5dd96e61686dbe788",
+	//        "devUtils": "0x38ef19fdf8e8415f18c307ed71967e19aac28ba1",
+	//        "erc20Proxy": "0x1dc4c1cefef38a777b15aa20260a54e584b16c48",
+	//        "erc721Proxy": "0x1d7022f5b17d2f8b695918fb48fa1089c9f85401"
+	//    }
+	//
+	CustomContractAddresses string `envvar:"CUSTOM_CONTRACT_ADDRESSES" default:""`
 }
 
 type snapshotInfo struct {
@@ -120,6 +137,7 @@ type App struct {
 	snapshotExpirationWatcher *expirationwatch.Watcher
 	muIdToSnapshotInfo        sync.Mutex
 	idToSnapshotInfo          map[string]snapshotInfo
+	messageHandler            *MessageHandler
 }
 
 func New(config Config) (*App, error) {
@@ -129,6 +147,13 @@ func New(config Config) (*App, error) {
 	log.SetLevel(log.Level(config.Verbosity))
 	log.AddHook(loghooks.NewKeySuffixHook())
 
+	// Add custom contract addresses if needed.
+	if config.CustomContractAddresses != "" {
+		if err := parseAndAddCustomContractAddresses(config.EthereumNetworkID, config.CustomContractAddresses); err != nil {
+			return nil, err
+		}
+	}
+
 	// Load private key and add peer ID hook.
 	privKeyPath := filepath.Join(config.DataDir, "keys", "privkey")
 	privKey, err := initPrivateKey(privKeyPath)
@@ -210,6 +235,9 @@ func New(config Config) (*App, error) {
 	if err != nil {
 		return nil, err
 	}
+	messageHandler := &MessageHandler{
+		nextOffset: 0,
+	}
 
 	app := &App{
 		config:                    config,
@@ -225,6 +253,7 @@ func New(config Config) (*App, error) {
 		meshMessageJSONSchema:     meshMessageJSONSchema,
 		snapshotExpirationWatcher: snapshotExpirationWatcher,
 		idToSnapshotInfo:          map[string]snapshotInfo{},
+		messageHandler:            messageHandler,
 	}
 
 	log.WithFields(map[string]interface{}{
@@ -659,25 +688,33 @@ func (app *App) GetStats() (*rpc.GetStatsResponse, error) {
 	if err != nil {
 		return nil, err
 	}
-	latestBlock := rpc.LatestBlock{
-		Number: int(latestBlockHeader.Number.Int64()),
-		Hash:   latestBlockHeader.Hash,
+	var latestBlock rpc.LatestBlock
+	if latestBlockHeader != nil {
+		latestBlock = rpc.LatestBlock{
+			Number: int(latestBlockHeader.Number.Int64()),
+			Hash:   latestBlockHeader.Hash,
+		}
 	}
 	notRemovedFilter := app.db.Orders.IsRemovedIndex.ValueFilter([]byte{0})
 	numOrders, err := app.db.Orders.NewQuery(notRemovedFilter).Count()
 	if err != nil {
 		return nil, err
 	}
+	numOrdersIncludingRemoved, err := app.db.Orders.Count()
+	if err != nil {
+		return nil, err
+	}
 
 	response := &rpc.GetStatsResponse{
-		Version:           version,
-		PubSubTopic:       getPubSubTopic(app.config.EthereumNetworkID),
-		Rendezvous:        getRendezvous(app.config.EthereumNetworkID),
-		PeerID:            app.peerID.String(),
-		EthereumNetworkID: app.config.EthereumNetworkID,
-		LatestBlock:       latestBlock,
-		NumOrders:         numOrders,
-		NumPeers:          app.node.GetNumPeers(),
+		Version:                   version,
+		PubSubTopic:               getPubSubTopic(app.config.EthereumNetworkID),
+		Rendezvous:                getRendezvous(app.config.EthereumNetworkID),
+		PeerID:                    app.peerID.String(),
+		EthereumNetworkID:         app.config.EthereumNetworkID,
+		LatestBlock:               latestBlock,
+		NumOrders:                 numOrders,
+		NumPeers:                  app.node.GetNumPeers(),
+		NumOrdersIncludingRemoved: numOrdersIncludingRemoved,
 	}
 	return response, nil
 }
@@ -698,13 +735,14 @@ func (app *App) periodicallyLogStats(ctx context.Context) {
 			continue
 		}
 		log.WithFields(log.Fields{
-			"version":           stats.Version,
-			"pubSubTopic":       stats.PubSubTopic,
-			"rendezvous":        stats.Rendezvous,
-			"ethereumNetworkID": stats.EthereumNetworkID,
-			"latestBlock":       stats.LatestBlock,
-			"numOrders":         stats.NumOrders,
-			"numPeers":          stats.NumPeers,
+			"version":                   stats.Version,
+			"pubSubTopic":               stats.PubSubTopic,
+			"rendezvous":                stats.Rendezvous,
+			"ethereumNetworkID":         stats.EthereumNetworkID,
+			"latestBlock":               stats.LatestBlock,
+			"numOrders":                 stats.NumOrders,
+			"numOrdersIncludingRemoved": stats.NumOrdersIncludingRemoved,
+			"numPeers":                  stats.NumPeers,
 		}).Info("current stats")
 	}
 }
@@ -714,3 +752,14 @@ func (app *App) SubscribeToOrderEvents(sink chan<- []*zeroex.OrderEvent) event.S
 	subscription := app.orderWatcher.Subscribe(sink)
 	return subscription
 }
+
+func parseAndAddCustomContractAddresses(networkID int, encodedContractAddresses string) error {
+	customAddresses := ethereum.ContractAddresses{}
+	if err := json.Unmarshal([]byte(encodedContractAddresses), &customAddresses); err != nil {
+		return fmt.Errorf("config.CustomContractAddresses is invalid: %s", err.Error())
+	}
+	if err := ethereum.AddContractAddressesForNetworkID(networkID, customAddresses); err != nil {
+		return fmt.Errorf("config.CustomContractAddresses is invalid: %s", err.Error())
+	}
+	return nil
+}

core/core.go

@@ -1,8 +1,6 @@
 package core
 
 import (
-	"math/rand"
-
 	"github.com/0xProject/0x-mesh/meshdb"
 	"github.com/0xProject/0x-mesh/p2p"
 	"github.com/0xProject/0x-mesh/zeroex"
@@ -14,8 +12,19 @@ import (
 // Ensure that App implements p2p.MessageHandler.
 var _ p2p.MessageHandler = &App{}
 
+type MessageHandler struct {
+	nextOffset int
+}
+
+func min(a int, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 func (app *App) GetMessagesToShare(max int) ([][]byte, error) {
-	// For now, we just select a random set of orders from those we have stored.
+	// For now, we use a round robin strategy to select a set of orders to share.
 	// We might return less than max even if there are max or greater orders
 	// currently stored.
 	// Use a snapshot to make sure state doesn't change between our two queries.
@@ -32,18 +41,37 @@ func (app *App) GetMessagesToShare(max int) ([][]byte, error) {
 	if count == 0 {
 		return nil, nil
 	}
-	// If count is less than max, we use an offset of 0 to simply return all
-	// orders we have stored.
-	offset := 0
-	if count > max {
-		// If count is greater than max, choose an offset such that we always return
-		// at least max orders.
-		offset = rand.Intn(count - max)
-	}
+
+	// Select up to the maximum number of orders starting at the offset that was
+	// calculated the last time this was called with `app`.
+	offset := min(app.messageHandler.nextOffset, count)
 	var selectedOrders []*meshdb.Order
-	err = ordersSnapshot.NewQuery(notRemovedFilter).Offset(offset).Max(max).Run(&selectedOrders)
-	if err != nil {
-		return nil, err
+	if offset != count {
+		err = ordersSnapshot.NewQuery(notRemovedFilter).Offset(offset).Max(max).Run(&selectedOrders)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// If more orders can be shared than were selected, append the maximum amount of
+	// unique (in this round) orders that can be added to the selected orders without
+	// exceeding the maximum number to share.
+	overflow := min(max-len(selectedOrders), offset)
+	if overflow > 0 {
+		var overflowSelectedOrders []*meshdb.Order
+		err = ordersSnapshot.NewQuery(notRemovedFilter).Offset(0).Max(overflow).Run(&overflowSelectedOrders)
+		if err != nil {
+			return nil, err
+		}
+		selectedOrders = append(selectedOrders, overflowSelectedOrders...)
+		app.messageHandler.nextOffset = overflow
+	} else {
+		// Calculate the next offset and wrap back to 0 if the next offset is larger
+		// than or equal to count.
+		app.messageHandler.nextOffset += max
+		if app.messageHandler.nextOffset >= count {
+			app.messageHandler.nextOffset = 0
+		}
 	}
 
 	log.WithFields(map[string]interface{}{