Release 10.2.0 (#993)

* Use write-ahead log for journal mode for sqlite

* test mutex and local approval

* fix critical errors during invalid ERC20 event spec

* Revert "Use write-ahead log for journal mode for sqlite"

This reverts commit 6048e09.

* remove unused marshaller

* use multiple datastore paths, add test for ERC20 approvals

* fix linter and comment

* add mutex around contractAddressToSeenCount map

this should solve the issue with panics around concurrent map read writes

* use the same defaults for newDB

* implement simple concurrent counter for addresses

* add configuration to resolver, dropping of slow subscribers

* version bump

* fix ts building

* exclude graphql_server.go from js builds

* cut release 10.2.0

Co-authored-by: Mason Liang <[email protected]>

README.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-10.1.0-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-10.2.0-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 [![Docs](https://img.shields.io/badge/docs-website-yellow.svg)](https://0x-org.gitbook.io/mesh)
 [![Chat with us on Discord](https://img.shields.io/badge/chat-Discord-blueViolet.svg)](https://discord.gg/HF7fHwk)
 [![Circle CI](https://img.shields.io/circleci/project/0xProject/0x-mesh/master.svg)](https://circleci.com/gh/0xProject/0x-mesh/tree/master)

cmd/mesh/graphql_server.go

@@ -1,3 +1,5 @@
+// +build !js
+
 package main
 
 import (
@@ -17,23 +19,25 @@ import (
 // the signal to shutdown.
 const gracefulShutdownTimeout = 10 * time.Second
 
-func serveGraphQL(ctx context.Context, app *core.App, addr string, enableGraphiQL bool) error {
+func serveGraphQL(ctx context.Context, app *core.App, config *standaloneConfig) error {
 	handler := http.NewServeMux()
 
 	// Set up handler for GraphiQL
-	if enableGraphiQL {
+	if config.EnableGraphQLServer {
 		handler.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			_, _ = w.Write(graphiQLPage)
 		}))
 	}
 
 	// Set up handler for GrqphQL queries
-	resolver := graphql.NewResolver(app)
+	resolver := graphql.NewResolver(app, &graphql.ResolverConfig{
+		SlowSubscriberTimeout: config.GraphQLSlowSubscriberTimeout,
+	})
 	graphQLServer := gqlserver.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
 	handler.Handle("/graphql", graphQLServer)
 
 	// Start the server
-	server := &http.Server{Addr: addr, Handler: handler}
+	server := &http.Server{Addr: config.GraphQLServerAddr, Handler: handler}
 	go func() {
 		<-ctx.Done()
 		shutdownContext, cancel := context.WithTimeout(context.Background(), gracefulShutdownTimeout)

cmd/mesh/main.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"os"
 	"sync"
+	"time"
 
 	"github.com/0xProject/0x-mesh/core"
 	"github.com/plaid/go-envvar/envvar"
@@ -29,6 +30,9 @@ type standaloneConfig struct {
 	// By default, 0x Mesh will listen on 0.0.0.0 (all available addresses) and
 	// port 60557.
 	GraphQLServerAddr string `envvar:"GRAPHQL_SERVER_ADDR" default:"0.0.0.0:60557"`
+	// GraphQLSlowSubscriberTimeout is the maximum amount of time subscriber has to
+	// accept events before being dropped.
+	GraphQLSlowSubscriberTimeout time.Duration `envvar:"GRAPHQL_SLOW_SUBSCRIBER_TIMEOUT" default:"2s"`
 	// EnableGraphQLPlayground determines whether or not to enable GraphiQL, an interactive
 	// GraphQL playground which can be accessed by visiting GraphQLServerAddr in a browser.
 	// See https://github.com/graphql/graphiql for more information. By default, GraphiQL
@@ -76,7 +80,7 @@ func main() {
 		go func() {
 			defer wg.Done()
 			log.WithField("graphql_server_addr", config.GraphQLServerAddr).Info("starting GraphQL server")
-			if err := serveGraphQL(ctx, app, config.GraphQLServerAddr, config.EnableGraphQLPlayground); err != nil {
+			if err := serveGraphQL(ctx, app, &config); err != nil {
 				graphQLErrChan <- err
 			}
 		}()

core/core.go

@@ -56,7 +56,7 @@ const (
 	estimatedNonPollingEthereumRPCRequestsPer24Hrs = 50000
 	// logStatsInterval is how often to log stats for this node.
 	logStatsInterval = 5 * time.Minute
-	version          = "10.1.0"
+	version          = "10.2.0"
 	// ordersyncMinPeers is the minimum amount of peers to receive orders from
 	// before considering the ordersync process finished.
 	ordersyncMinPeers = 5

core/new_db.go

@@ -10,10 +10,15 @@ import (
 )
 
 func newDB(ctx context.Context, config Config) (*db.DB, error) {
-	databasePath := filepath.Join(config.DataDir, "sqlite-db", "db.sqlite")
+	meshDatabasePath := filepath.Join(config.DataDir, "db", "db.sqlite?_journal=WAL")
+	peerStoreDatabasePath := filepath.Join(config.DataDir, "db", "peerstore.sqlite?_journal=WAL")
+	dhtDatabasePath := filepath.Join(config.DataDir, "db", "dht.sqlite?_journal=WAL")
+
 	return db.New(ctx, &db.Options{
-		DriverName:     "sqlite3",
-		DataSourceName: databasePath,
-		MaxOrders:      config.MaxOrdersInStorage,
+		DriverName:              "sqlite3",
+		DataSourceName:          meshDatabasePath,
+		DataSourcePeerStoreName: peerStoreDatabasePath,
+		DataSourceDHTName:       dhtDatabasePath,
+		MaxOrders:               config.MaxOrdersInStorage,
 	})
 }

db/common.go

@@ -50,10 +50,13 @@ type Database interface {
 }
 
 type Options struct {
-	DriverName     string `json:"driverName"`
-	DataSourceName string `json:"dataSourceName"`
-	MaxOrders      int    `json:"maxOrders"`
-	MaxMiniHeaders int    `json:"maxMiniHeaders"`
+	DriverName              string `json:"driverName"`
+	DataSourceName          string `json:"dataSourceName"`
+	DataSourcePeerStoreName string `json:"dataSourcePeerStoreName"`
+	DataSourceDHTName       string `json:"dataSourceDHTName"`
+	DataDir                 string `json:"dataDir"`
+	MaxOrders               int    `json:"maxOrders"`
+	MaxMiniHeaders          int    `json:"maxMiniHeaders"`
 }
 
 func parseOptions(opts *Options) *Options {

db/sql_implementation.go

@@ -20,8 +20,9 @@ import (
 	"github.com/google/uuid"
 	"github.com/ido50/sqlz"
 	ds "github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-ds-sql"
+	sqlds "github.com/ipfs/go-ds-sql"
 	"github.com/jmoiron/sqlx"
+
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -34,7 +35,13 @@ var _ Database = (*DB)(nil)
 type DB struct {
 	ctx   context.Context
 	sqldb *sqlz.DB
-	opts  *Options
+	// Additional connection contexts are created to avoid the `database is
+	// locked` issue when libp2p accesses the peerstore and DHT tables
+	// concurrently over the same connection. ref:
+	// https://github.com/0xProject/0x-mesh/pull/873
+	dhtSQLdb  *sqlz.DB
+	peerSQLdb *sqlz.DB
+	opts      *Options
 
 	// mu is used to protect all reads and writes to the database. This is a solution to the
 	// `database is locked` error that appears on SQLite. https://github.com/mattn/go-sqlite3/issues/607
@@ -44,20 +51,25 @@ type DB struct {
 
 func defaultOptions() *Options {
 	return &Options{
-		DriverName:     "sqlite3",
-		DataSourceName: "0x_mesh/db/db.sqlite",
-		MaxOrders:      100000,
-		MaxMiniHeaders: 20,
+		DriverName:              "sqlite3",
+		DataSourceName:          "0x_mesh/db/db.sqlite",
+		DataSourceDHTName:       "0x_mesh/db/dht.sqlite",
+		DataSourcePeerStoreName: "0x_mesh/db/peerstore.sqlite",
+		MaxOrders:               100000,
+		MaxMiniHeaders:          20,
 	}
 }
 
 // TestOptions returns a set of options suitable for testing.
 func TestOptions() *Options {
+	testingDir := filepath.Join("/tmp", "mesh_testing", uuid.New().String())
 	return &Options{
-		DriverName:     "sqlite3",
-		DataSourceName: filepath.Join("/tmp", "mesh_testing", uuid.New().String(), "db.sqlite"),
-		MaxOrders:      100,
-		MaxMiniHeaders: 20,
+		DriverName:              "sqlite3",
+		DataSourceName:          filepath.Join(testingDir, "db.sqlite"),
+		DataSourceDHTName:       filepath.Join(testingDir, "dht.sqlite"),
+		DataSourcePeerStoreName: filepath.Join(testingDir, "peerstore.sqlite"),
+		MaxOrders:               100,
+		MaxMiniHeaders:          20,
 	}
 }
 
@@ -69,25 +81,42 @@ func New(ctx context.Context, opts *Options) (*DB, error) {
 	connectCtx, cancel := context.WithTimeout(ctx, connectTimeout)
 	defer cancel()
 
-	if err := os.MkdirAll(filepath.Dir(opts.DataSourceName), os.ModePerm); err != nil && err != os.ErrExist {
-		return nil, err
+	pathsToCreate := []string{opts.DataSourceName, opts.DataSourceDHTName, opts.DataSourcePeerStoreName}
+	for _, path := range pathsToCreate {
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil && err != os.ErrExist {
+			return nil, err
+		}
 	}
 
 	sqldb, err := sqlx.ConnectContext(connectCtx, opts.DriverName, opts.DataSourceName)
 	if err != nil {
 		return nil, err
 	}
 
-	// Automatically close the database connection when the context is canceled.
+	dhtSQLdb, err := sqlx.ConnectContext(connectCtx, opts.DriverName, opts.DataSourceDHTName)
+	if err != nil {
+		return nil, err
+	}
+
+	peerSQLdb, err := sqlx.ConnectContext(connectCtx, opts.DriverName, opts.DataSourcePeerStoreName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Automatically close the database connections when the context is canceled.
 	go func() {
 		<-ctx.Done()
 		_ = sqldb.Close()
+		_ = dhtSQLdb.Close()
+		_ = peerSQLdb.Close()
 	}()
 
 	db := &DB{
-		ctx:   ctx,
-		sqldb: sqlz.Newx(sqldb),
-		opts:  opts,
+		ctx:       ctx,
+		sqldb:     sqlz.Newx(sqldb),
+		dhtSQLdb:  sqlz.Newx(dhtSQLdb),
+		peerSQLdb: sqlz.Newx(peerSQLdb),
+		opts:      opts,
 	}
 	if err := db.migrate(); err != nil {
 		return nil, err
@@ -97,11 +126,11 @@ func New(ctx context.Context, opts *Options) (*DB, error) {
 }
 
 func (db *DB) DHTStore() ds.Batching {
-	return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("dhtstore"))
+	return sqlds.NewDatastore(db.dhtSQLdb.DB.DB, NewSqliteQueriesForTable("dhtstore"))
 }
 
 func (db *DB) PeerStore() ds.Batching {
-	return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("peerstore"))
+	return sqlds.NewDatastore(db.peerSQLdb.DB.DB, NewSqliteQueriesForTable("peerstore"))
 }
 
 // TODO(albrow): Use a proper migration tool. We don't technically need this
@@ -157,12 +186,15 @@ CREATE TABLE IF NOT EXISTS metadata (
 	ethRPCRequestsSentInCurrentUTCDay BIGINT NOT NULL,
 	startOfCurrentUTCDay              DATETIME NOT NULL
 );
+`
 
+const peerstoreSchema = `
 CREATE TABLE IF NOT EXISTS peerstore (
 	key  TEXT NOT NULL UNIQUE,
 	data BYTEA NOT NULL
 );
-
+`
+const dhtSchema = `
 CREATE TABLE IF NOT EXISTS dhtstore (
 	key  TEXT NOT NULL UNIQUE,
 	data BYTEA NOT NULL
@@ -307,7 +339,18 @@ const updateMetadataQuery = `UPDATE metadata SET
 
 func (db *DB) migrate() error {
 	_, err := db.sqldb.ExecContext(db.ctx, schema)
-	return convertErr(err)
+	if err != nil {
+		return fmt.Errorf("meshdb schema migration failed with err: %s", err)
+	}
+	_, err = db.peerSQLdb.ExecContext(db.ctx, peerstoreSchema)
+	if err != nil {
+		return fmt.Errorf("peerstore schema migration failed with err: %s", err)
+	}
+	_, err = db.dhtSQLdb.ExecContext(db.ctx, dhtSchema)
+	if err != nil {
+		return fmt.Errorf("dht schema migration failed with err: %s", err)
+	}
+	return nil
 }
 
 // ReadWriteTransactionalContext acquires a write lock, executes the transaction, then immediately releases the lock.

docs/browser-bindings/browser-lite/README.md

@@ -1,4 +1,4 @@
-# @0x/mesh-browser-lite - v10.1.0
+# @0x/mesh-browser-lite - v10.2.0
 
 ## @0x/mesh-browser-lite