Merge pull request #264 from 0xProject/release/1.0.2-beta

Release version 1.0.2-beta

DEPLOYMENT.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-1.0.1--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-1.0.2--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 
 # 0x Mesh Deployment Guide
 

DEVELOPMENT.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-1.0.1--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-1.0.2--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 
 # 0x Mesh Development Guide
 

README.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-1.0.1--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-1.0.2--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 [![Chat with us on Discord](https://img.shields.io/badge/chat-Discord-blueViolet.svg)](https://discord.gg/HF7fHwk)
 [![GoDoc](https://godoc.org/github.com/0xProject/0x-mesh?status.svg)](https://godoc.org/github.com/0xProject/0x-mesh)
 [![Circle CI](https://img.shields.io/circleci/project/0xProject/0x-mesh/master.svg)](https://circleci.com/gh/0xProject/0x-mesh/tree/master)
@@ -12,7 +12,7 @@
 
 ## Versions
 
-You are looking at the documentation for version `1.0.1-beta`. To see the
+You are looking at the documentation for version `1.0.2-beta`. To see the
 documentation for a different version, visit the
 [Releases Page](https://github.com/0xProject/0x-mesh/releases).
 

USAGE.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-1.0.1--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-1.0.2--beta-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 
 # 0x Mesh Usage Guide
 

cmd/mesh-bootstrap/main.go

@@ -11,14 +11,13 @@ import (
 	"os"
 	"time"
 
-	autonat "github.com/libp2p/go-libp2p-autonat-svc"
-
 	"github.com/0xProject/0x-mesh/keys"
+	"github.com/0xProject/0x-mesh/loghooks"
 	"github.com/0xProject/0x-mesh/p2p"
 	libp2p "github.com/libp2p/go-libp2p"
+	autonat "github.com/libp2p/go-libp2p-autonat-svc"
 	connmgr "github.com/libp2p/go-libp2p-connmgr"
 	p2pcrypto "github.com/libp2p/go-libp2p-crypto"
-	dht "github.com/libp2p/go-libp2p-kad-dht"
 	p2pnet "github.com/libp2p/go-libp2p-net"
 	peer "github.com/libp2p/go-libp2p-peer"
 	ma "github.com/multiformats/go-multiaddr"
@@ -91,6 +90,9 @@ func main() {
 		log.WithField("error", err).Fatal("could not create host")
 	}
 
+	// Add the peer ID hook to the logger.
+	log.AddHook(loghooks.NewPeerIDHook(basicHost.ID()))
+
 	// Set up the notifee.
 	basicHost.Network().Notify(&notifee{})
 
@@ -100,7 +102,7 @@ func main() {
 	}
 
 	// Set up DHT for peer discovery.
-	kadDHT, err := dht.New(ctx, basicHost)
+	kadDHT, err := p2p.NewDHT(ctx, basicHost)
 	if err != nil {
 		log.WithField("error", err).Fatal("could not create DHT")
 	}
@@ -128,8 +130,7 @@ func main() {
 	}
 
 	log.WithFields(map[string]interface{}{
-		"addrs":  basicHost.Addrs(),
-		"peerID": basicHost.ID(),
+		"addrs": basicHost.Addrs(),
 	}).Info("started bootstrap node")
 
 	// Sleep until stopped
@@ -164,17 +165,17 @@ func (n *notifee) ListenClose(p2pnet.Network, ma.Multiaddr) {}
 // Connected is called when a connection opened
 func (n *notifee) Connected(network p2pnet.Network, conn p2pnet.Conn) {
 	log.WithFields(map[string]interface{}{
-		"peerID":       conn.RemotePeer(),
-		"multiaddress": conn.RemoteMultiaddr(),
-	}).Trace("connected to peer")
+		"remotePeerID":       conn.RemotePeer(),
+		"remoteMultiaddress": conn.RemoteMultiaddr(),
+	}).Info("connected to peer")
 }
 
 // Disconnected is called when a connection closed
 func (n *notifee) Disconnected(network p2pnet.Network, conn p2pnet.Conn) {
 	log.WithFields(map[string]interface{}{
-		"peerID":       conn.RemotePeer(),
-		"multiaddress": conn.RemoteMultiaddr(),
-	}).Trace("disconnected from peer")
+		"remotePeerID":       conn.RemotePeer(),
+		"remoteMultiaddress": conn.RemoteMultiaddr(),
+	}).Info("disconnected from peer")
 }
 
 // OpenedStream is called when a stream opened

core/core.go

@@ -17,6 +17,7 @@ import (
 	"github.com/0xProject/0x-mesh/ethereum/blockwatch"
 	"github.com/0xProject/0x-mesh/expirationwatch"
 	"github.com/0xProject/0x-mesh/keys"
+	"github.com/0xProject/0x-mesh/loghooks"
 	"github.com/0xProject/0x-mesh/meshdb"
 	"github.com/0xProject/0x-mesh/p2p"
 	"github.com/0xProject/0x-mesh/rpc"
@@ -106,7 +107,7 @@ func New(config Config) (*App, error) {
 	log.SetLevel(log.Level(config.Verbosity))
 	log.WithFields(map[string]interface{}{
 		"config":  config,
-		"version": "1.0.1-beta",
+		"version": "1.0.2-beta",
 	}).Info("Initializing new core.App")
 
 	if config.EthereumRPCMaxContentLength < maxOrderSizeInBytes {
@@ -218,6 +219,9 @@ func New(config Config) (*App, error) {
 	}
 	app.node = node
 
+	// Add the peer ID hook to the logger.
+	log.AddHook(loghooks.NewPeerIDHook(node.ID()))
+
 	return app, nil
 }
 
@@ -255,7 +259,6 @@ func (app *App) Start() error {
 	go app.periodicallyCheckForNewAddrs(addrs)
 	log.WithFields(map[string]interface{}{
 		"addresses": addrs,
-		"peerID":    app.node.ID().String(),
 	}).Info("started p2p node")
 
 	// TODO(albrow) we might want to match the synchronous API of p2p.Node which

core/message_handler.go

@@ -129,11 +129,15 @@ func (app *App) HandleMessages(messages []*p2p.Message) error {
 		// If we've reached this point, the message is valid and we were able to
 		// decode it into an order. Append it to the list of orders to validate and
 		// update peer scores accordingly.
+		log.WithFields(map[string]interface{}{
+			"orderHash": orderHash,
+			"from":      msg.From.String(),
+		}).Info("received new valid order from peer")
 		log.WithFields(map[string]interface{}{
 			"order":     order,
 			"orderHash": orderHash,
 			"from":      msg.From.String(),
-		}).Trace("received order from peer")
+		}).Trace("all fields for new valid order received from peer")
 		orders = append(orders, order)
 		orderHashToMessage[orderHash] = msg
 		app.handlePeerScoreEvent(msg.From, psValidMessage)

examples/beta_telemetry_node/README.md

@@ -0,0 +1,99 @@
+## Deploying a Telemetry-Enabled Mesh Node
+
+0x Mesh is completely permissionless and the beta is open to anyone who wants to
+participate. You can optionally help improve 0x Mesh by enabling telemetry. Mesh
+automatically logs a lot of useful information including the number of orders
+processed and details about any errors and warnings that might occur. Sending
+this information to us is extraordinarily helpful, but completely optional. If
+you don't want to enable telemetry, you can follow the
+[Deployment Guide](../../DEPLOYMENT.md) instead.
+
+This guide will walk you though setting up a telemetry-enabled Mesh node on the cloud hosting solution of your choice using [Docker Machine](https://docs.docker.com/machine/). The instructions below will deploy a Mesh node on [DigitalOcean](https://www.digitalocean.com/), but can be easily modified to deploy on [many other cloud providers](https://docs.docker.com/machine/drivers/).
+
+## Prerequisites
+
+-   [Docker](https://www.docker.com/get-started)
+-   [Docker Machine](https://docs.docker.com/machine/install-machine/) On Mac/Windows it is installed with Docker Desktop.
+-   [Docker Compose](https://docs.docker.com/compose/install/) On Mac, it is installed with Docker Desktop.
+
+## Instructions
+
+Let's start by cloning the Mesh repo and navigating to the `beta_telemetry_node` directory:
+
+```bash
+git clone https://github.com/0xProject/0x-mesh.git
+cd 0x-mesh/examples/beta_telemetry_node
+code . # open this directory in your text editor
+```
+
+Please open up `docker-compose.yml` and set `ETHEREUM_RPC_URL` to your own Ethereum JSON RPC endpoint.
+
+We are now ready to deploy our instance. Before we can continue, you will need to set up an account with the cloud hosting provider of your choice, and retrieve your access token/key/secret. We will use them to create a new machine with name `mesh-beta`. Docker has great documentation on doing all of that for [DigitalOcean](https://docs.docker.com/machine/examples/ocean/) and [AWS](https://docs.docker.com/machine/examples/aws/). Instead of naming the machine `docker-sandbox` as in those examples, let's name ours `mesh-beta` as shown below.
+
+```bash
+docker-machine create --driver digitalocean --digitalocean-access-token xxxxx mesh-beta
+```
+
+Make sure you replaced `xxxxx` with your access token. This command will spin up a new instance on your cloud provider, pre-installed with Docker so that it's ready-to-use with the `docker-machine` command. Once the command completes, let's make sure the machine exists:
+
+```bash
+docker-machine ls
+```
+
+You should see something like:
+
+```
+mesh-beta     -        digitalocean   Running   tcp://162.31.121.332:2376            v18.09.7
+```
+
+Our remote machine is alive! The next step is to copy over two config files to this remote machine. We can use [docker-machine scp](https://docs.docker.com/machine/reference/scp/) to do this. The following commands ask `docker-machine` to copy over the `fluent-bit.conf` and `parsers.conf` files from our local computer to a directory called `root` on the `mesh-beta` machine.
+
+```bash
+docker-machine scp fluent-bit.conf mesh-beta:/root/fluent-bit.conf
+docker-machine scp parsers.conf mesh-beta:/root/parsers.conf
+```
+
+Now comes the Docker Machine magic. By running the following commands, we can ask Docker Machine to let us execute any Docker command in our local shell AS IF we were executing them directly on the `mesh-beta` machine:
+
+```bash
+docker-machine env mesh-beta
+eval $(docker-machine env mesh-beta)
+```
+
+Presto! We are now ready to spin up our telemetry-enabled Mesh node! We do this using the Docker Compose command `up`:
+
+```bash
+docker-compose up -d
+```
+
+Houston, we have lift-off! All the logs from the Mesh node are being piped to the [Fluentbit](https://fluentbit.io/) instance that got deployed alongside Mesh. So if you want to inspect the logs, you need to do:
+
+```bash
+docker logs <fluent-bit-container-id> -f
+```
+
+Instead of reading them from the `0xorg/mesh` container.
+
+Finally, in order to prevent our log aggregation stack from getting overloaded,
+we whitelist the peers that are allowed to send us logs. Look for a log message
+that looks like this:
+
+```json
+{
+    "addresses": ["/ip4/127.0.0.1/tcp/60557", "/ip4/172.17.0.2/tcp/60557"],
+    "level": "info",
+    "msg": "started p2p node",
+    "peerID": "QmbKkHnmkmFxKbPWbBNz3inKizDuqjTsWsVyutnshYULLp",
+    "time": "2019-07-15T17:36:46-07:00"
+}
+```
+
+Ping us in [Discord](https://discord.gg/HF7fHwk) and let us know your peer ID. You can DM `fabio#1058`, `Alex Browne | 0x#2975` or `ovrmrrw#0454` and we'll whitelist your node :)
+
+I hope that was easy enough! If you ran into any issues, please ping us in the #mesh channel on [Discord](https://discord.gg/HF7fHwk). To learn more about connecting to your Mesh node's JSON RPC interface, check out our [Usage docs](../../USAGE.md). Your node's JSON RPC endpoint should be available at `ws://<your-ip-address>:60557` and you can discover your machine's IP address by running:
+
+```
+docker-machine ip mesh-beta
+```
+
+Cheers!

examples/beta_telemetry_node/docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3'
+
+services:
+    mesh:
+        image: 0xorg/mesh:1.0.2-beta
+        restart: always
+        logging:
+            driver: fluentd
+            options:
+                fluentd-address: localhost:24224
+                tag: docker.mesh
+        volumes:
+            - /root/data:/usr/mesh/0x_mesh
+        links:
+            - fluentbit
+        ports:
+            - '60557:60557'
+            - '60558:60558'
+        environment:
+            - VERBOSITY=5
+            - P2P_LISTEN_PORT=60558
+            - ETHEREUM_NETWORK_ID=1
+            # Set your backing Ethereum JSON RPC endpoint below
+            - ETHEREUM_RPC_URL=
+            - BLOCK_POLLING_INTERVAL=5s
+
+    fluentbit:
+        image: fluent/fluent-bit:1.2.1
+        links:
+            - esproxy
+        ports:
+            - '24224:24224'
+        command: /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
+        volumes:
+            - /root/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+            - /root/parsers.conf:/fluent-bit/etc/parsers.conf
+    esproxy:
+        image: overmorrow/auth-es-proxy:latest
+        ports:
+            - '3333:3333'
+        volumes:
+            - /root/data/keys:/app/keys
+        restart: on-failure:5
+        environment:
+            - PORT=3333
+            - REMOTE_ADDRESS=https://telemetry.mesh.0x.org/_bulk
+            - INPUT_VALIDATION=false
+            - OUTPUT_SIGNING=true
+            - PRIVATE_KEY_PATH=/app/keys/privkey

examples/beta_telemetry_node/fluent-bit.conf

@@ -0,0 +1,32 @@
+[SERVICE]
+    Flush        5
+    Daemon       Off
+    Log_Level    debug
+    Parsers_File /fluent-bit/etc/parsers.conf
+
+[INPUT]
+    Name      forward
+    Listen    fluentbit
+    Port      24224
+
+[FILTER]
+    Name parser
+    Match docker.mesh
+    Key_Name log
+    Parser dockerCustom
+    Unescape_Key On
+    Preserve_Key On
+
+[OUTPUT]
+    Name  es
+    Match *
+    Host  esproxy
+    Port  3333
+    Logstash_Format On
+    Logstash_Prefix mesh_beta
+    Type  docker
+
+[OUTPUT]
+    Name  stdout
+    Match *
+    Format msgpack

examples/beta_telemetry_node/parsers.conf

@@ -0,0 +1,8 @@
+[PARSER]
+    Name        dockerCustom
+    Format      json
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S %z
+    Decode_Field_As escaped_utf8 log do_next
+    Decode_Field_As escaped log do_next
+    Decode_Field_As json log

loghooks/peer_id_hook.go

@@ -0,0 +1,29 @@
+package loghooks
+
+import (
+	peer "github.com/libp2p/go-libp2p-peer"
+	log "github.com/sirupsen/logrus"
+)
+
+// PeerIDHook is a logger hook that injects the peer ID in all logs when
+// possible.
+type PeerIDHook struct {
+	peerID string
+}
+
+// NewPeerIDHook creates and returns a new PeerIDHook with the given peer ID.
+func NewPeerIDHook(peerID peer.ID) *PeerIDHook {
+	return &PeerIDHook{peerID: peerID.String()}
+}
+
+// Ensure that PeerIDHook implements log.Hook.
+var _ log.Hook = &PeerIDHook{}
+
+func (h *PeerIDHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (h *PeerIDHook) Fire(entry *log.Entry) error {
+	entry.Data["myPeerID"] = h.peerID
+	return nil
+}