Merge pull request #954 from 0xProject/release/10.1.0

Release 10.1.0

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 This changelog is a work in progress and may contain notes for versions which have not actually been released. Check the [Releases](https://github.com/0xProject/0x-mesh/releases) page to see full release notes and more information about the latest released versions.
 
+## v10.1.0
+
+### Features ✅
+
+-   Added support for continuing to watch and store orders that become unfillable. [#945](https://github.com/0xProject/0x-mesh/pull/945)
+
 ## v10.0.0
 
 ### Breaking changes 🛠

README.md

@@ -1,4 +1,4 @@
-[![Version](https://img.shields.io/badge/version-10.0.0-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
+[![Version](https://img.shields.io/badge/version-10.1.0-orange.svg)](https://github.com/0xProject/0x-mesh/releases)
 [![Docs](https://img.shields.io/badge/docs-website-yellow.svg)](https://0x-org.gitbook.io/mesh)
 [![Chat with us on Discord](https://img.shields.io/badge/chat-Discord-blueViolet.svg)](https://discord.gg/HF7fHwk)
 [![Circle CI](https://img.shields.io/circleci/project/0xProject/0x-mesh/master.svg)](https://circleci.com/gh/0xProject/0x-mesh/tree/master)

common/types/types.go

@@ -55,6 +55,18 @@ type AddOrdersOpts struct {
 	// and will always stay in storage until they are no longer fillable. Defaults
 	// to true.
 	Pinned bool `json:"pinned"`
+	// KeepCancelled signals that this order should not be deleted
+	// even if it is cancelled.
+	KeepCancelled bool `json:"keepCancelled"`
+	// KeepExpired signals that this order should not be deleted
+	// even if it becomes expired.
+	KeepExpired bool `json:"keepExpired"`
+	// KeepFullyFilled signals that this order should not be deleted
+	// even if it is fully filled.
+	KeepFullyFilled bool `json:"keepFullyFilled"`
+	// KeepUnfunded signals that this order should not be deleted
+	// even if it becomes unfunded.
+	KeepUnfunded bool `json:"keepUnfunded"`
 }
 
 // OrderInfo represents an fillable order and how much it could be filled for.
@@ -126,6 +138,10 @@ type OrderWithMetadata struct {
 	// IsPinned indicates whether or not the order is pinned. Pinned orders are
 	// not removed from the database unless they become unfillable.
 	IsPinned bool `json:"isPinned"`
+	// IsUnfillable indicates whether or not the order has become unfillable.
+	IsUnfillable bool `json:"isUnfillable"`
+	// IsExpired indicates whether or not the order has become expired.
+	IsExpired bool `json:"isExpired"`
 	// JSON-encoded list of assetdatas contained in MakerAssetData. For non-MAP
 	// orders, the list contains only one element which is equal to MakerAssetData.
 	// For MAP orders, it contains each component assetdata.
@@ -138,6 +154,18 @@ type OrderWithMetadata struct {
 	// LastValidatedBlockHash is the hash of the block at which the order was
 	// last validated.
 	LastValidatedBlockHash common.Hash `json:"lastValidatedBlockHash"`
+	// KeepCancelled signals that this order should not be deleted
+	// if it is cancelled.
+	KeepCancelled bool `json:"keepCancelled"`
+	// KeepExpired signals that this order should not be deleted
+	// if it becomes expired.
+	KeepExpired bool `json:"keepExpired"`
+	// KeepFullyFilled signals that this order should not be deleted
+	// if it is fully filled.
+	KeepFullyFilled bool `json:"keepFullyFilled"`
+	// KeepUnfunded signals that this order should not be deleted
+	// if it becomes unfunded.
+	KeepUnfunded bool `json:"keepUnfunded"`
 }
 
 func (order OrderWithMetadata) SignedOrder() *zeroex.SignedOrder {

core/core.go

@@ -56,7 +56,7 @@ const (
 	estimatedNonPollingEthereumRPCRequestsPer24Hrs = 50000
 	// logStatsInterval is how often to log stats for this node.
 	logStatsInterval = 5 * time.Minute
-	version          = "10.0.0"
+	version          = "10.1.0"
 	// ordersyncMinPeers is the minimum amount of peers to receive orders from
 	// before considering the ordersync process finished.
 	ordersyncMinPeers = 5
@@ -866,7 +866,7 @@ func (app *App) GetOrders(perPage int, minOrderHash common.Hash) (*types.GetOrde
 // is true, the orders will be marked as pinned, which means they will only be
 // removed if they become unfillable and will not be removed due to having a high
 // expiration time or any incentive mechanisms.
-func (app *App) AddOrders(ctx context.Context, signedOrders []*zeroex.SignedOrder, pinned bool) (*ordervalidator.ValidationResults, error) {
+func (app *App) AddOrders(ctx context.Context, signedOrders []*zeroex.SignedOrder, pinned bool, opts *types.AddOrdersOpts) (*ordervalidator.ValidationResults, error) {
 	signedOrdersRaw := []*json.RawMessage{}
 	buf := &bytes.Buffer{}
 	if err := json.NewEncoder(buf).Encode(signedOrders); err != nil {
@@ -875,11 +875,11 @@ func (app *App) AddOrders(ctx context.Context, signedOrders []*zeroex.SignedOrde
 	if err := json.NewDecoder(buf).Decode(&signedOrdersRaw); err != nil {
 		return nil, err
 	}
-	return app.AddOrdersRaw(ctx, signedOrdersRaw, pinned)
+	return app.AddOrdersRaw(ctx, signedOrdersRaw, pinned, opts)
 }
 
 // AddOrdersRaw is like AddOrders but accepts raw JSON messages.
-func (app *App) AddOrdersRaw(ctx context.Context, signedOrdersRaw []*json.RawMessage, pinned bool) (*ordervalidator.ValidationResults, error) {
+func (app *App) AddOrdersRaw(ctx context.Context, signedOrdersRaw []*json.RawMessage, pinned bool, opts *types.AddOrdersOpts) (*ordervalidator.ValidationResults, error) {
 	<-app.started
 
 	allValidationResults := &ordervalidator.ValidationResults{
@@ -944,7 +944,7 @@ func (app *App) AddOrdersRaw(ctx context.Context, signedOrdersRaw []*json.RawMes
 		orderHashesSeen[orderHash] = struct{}{}
 	}
 
-	validationResults, err := app.orderWatcher.ValidateAndStoreValidOrders(ctx, schemaValidOrders, pinned, app.chainID)
+	validationResults, err := app.orderWatcher.ValidateAndStoreValidOrders(ctx, schemaValidOrders, app.chainID, pinned, opts)
 	if err != nil {
 		return nil, err
 	}

core/core_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/0xProject/0x-mesh/common/types"
 	"github.com/0xProject/0x-mesh/constants"
 	"github.com/0xProject/0x-mesh/db"
 	"github.com/0xProject/0x-mesh/ethereum"
@@ -287,7 +288,7 @@ func runOrdersyncTestCase(testCase ordersyncTestCase) func(t *testing.T) {
 		// We have to wait for latest block to be processed by the Mesh node.
 		time.Sleep(blockProcessingWaitTime)
 
-		results, err := originalNode.orderWatcher.ValidateAndStoreValidOrders(ctx, originalOrders, true, constants.TestChainID)
+		results, err := originalNode.orderWatcher.ValidateAndStoreValidOrders(ctx, originalOrders, constants.TestChainID, true, &types.AddOrdersOpts{})
 		require.NoError(t, err)
 		require.Empty(t, results.Rejected, "tried to add orders but some were invalid: \n%s\n", spew.Sdump(results))
 

core/message_handler.go

@@ -3,6 +3,7 @@ package core
 import (
 	"context"
 
+	"github.com/0xProject/0x-mesh/common/types"
 	"github.com/0xProject/0x-mesh/constants"
 	"github.com/0xProject/0x-mesh/encoding"
 	"github.com/0xProject/0x-mesh/p2p"
@@ -56,7 +57,7 @@ func (app *App) HandleMessages(ctx context.Context, messages []*p2p.Message) err
 	}
 
 	// Next, we validate the orders.
-	validationResults, err := app.orderWatcher.ValidateAndStoreValidOrders(ctx, orders, false, app.chainID)
+	validationResults, err := app.orderWatcher.ValidateAndStoreValidOrders(ctx, orders, app.chainID, false, &types.AddOrdersOpts{})
 	if err != nil {
 		return err
 	}

core/ordersync_subprotocols.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/0xProject/0x-mesh/common/types"
 	"github.com/0xProject/0x-mesh/core/ordersync"
 	"github.com/0xProject/0x-mesh/orderfilter"
 	"github.com/0xProject/0x-mesh/zeroex"
@@ -170,7 +171,7 @@ func (p *FilteredPaginationSubProtocolV0) HandleOrderSyncResponse(ctx context.Co
 			p.app.handlePeerScoreEvent(res.ProviderID, psReceivedOrderDoesNotMatchFilter)
 		}
 	}
-	validationResults, err := p.app.orderWatcher.ValidateAndStoreValidOrders(ctx, filteredOrders, false, p.app.chainID)
+	validationResults, err := p.app.orderWatcher.ValidateAndStoreValidOrders(ctx, filteredOrders, p.app.chainID, false, &types.AddOrdersOpts{})
 	if err != nil {
 		return nil, len(filteredOrders), err
 	}
@@ -361,7 +362,7 @@ func (p *FilteredPaginationSubProtocolV1) HandleOrderSyncResponse(ctx context.Co
 			p.app.handlePeerScoreEvent(res.ProviderID, psReceivedOrderDoesNotMatchFilter)
 		}
 	}
-	validationResults, err := p.app.orderWatcher.ValidateAndStoreValidOrders(ctx, filteredOrders, false, p.app.chainID)
+	validationResults, err := p.app.orderWatcher.ValidateAndStoreValidOrders(ctx, filteredOrders, p.app.chainID, false, &types.AddOrdersOpts{})
 	if err != nil {
 		return nil, len(filteredOrders), err
 	}

db/common.go

@@ -117,9 +117,15 @@ const (
 	OFFillableTakerAssetAmount OrderField = "fillableTakerAssetAmount"
 	OFIsRemoved                OrderField = "isRemoved"
 	OFIsPinned                 OrderField = "isPinned"
+	OFIsUnfillable             OrderField = "isUnfillable"
+	OFIsExpired                OrderField = "isExpired"
 	OFParsedMakerAssetData     OrderField = "parsedMakerAssetData"
 	OFParsedMakerFeeAssetData  OrderField = "parsedMakerFeeAssetData"
 	OFLastValidatedBlockNumber OrderField = "lastValidatedBlockNumber"
+	OFKeepCancelled            OrderField = "keepCancelled"
+	OFKeepExpired              OrderField = "keepExpired"
+	OFKeepFullyFilled          OrderField = "keepFullyFilled"
+	OFKeepUnfunded             OrderField = "keepUnfunded"
 )
 
 type OrderQuery struct {
@@ -143,6 +149,7 @@ type OrderFilter struct {
 type StoredOrderStatus struct {
 	IsStored                 bool     `json:"isStored"`
 	IsMarkedRemoved          bool     `json:"isMarkedRemoved"`
+	IsMarkedUnfillable       bool     `json:"isMarkedUnfillable"`
 	FillableTakerAssetAmount *big.Int `json:"fillableTakerAssetAmount"`
 }
 

db/db_test.go

@@ -207,32 +207,36 @@ func TestGetOrderStatuses(t *testing.T) {
 
 	removedOrder := newTestOrder()
 	removedOrder.IsRemoved = true
-	notRemovedOrder := newTestOrder()
-	_, _, _, err := db.AddOrders([]*types.OrderWithMetadata{removedOrder, notRemovedOrder})
+	unfillableOrder := newTestOrder()
+	unfillableOrder.IsUnfillable = true
+	_, _, _, err := db.AddOrders([]*types.OrderWithMetadata{removedOrder, unfillableOrder})
 	require.NoError(t, err)
 
 	hashes := []common.Hash{
 		common.HexToHash("0xace746910c6a8a4730878e6e8a4abb328844c0b58f0cdfbb5b6ad28ee0bae347"),
 		removedOrder.Hash,
-		notRemovedOrder.Hash,
+		unfillableOrder.Hash,
 	}
 	actualStatuses, err := db.GetOrderStatuses(hashes)
 	require.NoError(t, err)
 	expectedStatuses := []*StoredOrderStatus{
 		{
 			IsStored:                 false,
 			IsMarkedRemoved:          false,
+			IsMarkedUnfillable:       false,
 			FillableTakerAssetAmount: nil,
 		},
 		{
 			IsStored:                 true,
 			IsMarkedRemoved:          true,
+			IsMarkedUnfillable:       false,
 			FillableTakerAssetAmount: removedOrder.FillableTakerAssetAmount,
 		},
 		{
 			IsStored:                 true,
 			IsMarkedRemoved:          false,
-			FillableTakerAssetAmount: notRemovedOrder.FillableTakerAssetAmount,
+			IsMarkedUnfillable:       true,
+			FillableTakerAssetAmount: unfillableOrder.FillableTakerAssetAmount,
 		},
 	}
 	assert.Equal(t, expectedStatuses, actualStatuses)
@@ -1239,6 +1243,8 @@ func newTestOrder() *types.OrderWithMetadata {
 		FillableTakerAssetAmount: big.NewInt(42),
 		IsRemoved:                false,
 		IsPinned:                 true,
+		IsUnfillable:             false,
+		IsExpired:                false,
 		ParsedMakerAssetData: []*types.SingleAssetData{
 			{
 				Address: constants.GanacheDummyERC721TokenAddress,
@@ -1373,6 +1379,39 @@ func makeOrderFilterTestCases(t *testing.T, db *DB) ([]*types.OrderWithMetadata,
 			},
 			expectedMatchingOrders: storedOrders,
 		},
+		{
+			name: "IsPinned = true",
+			filters: []OrderFilter{
+				{
+					Field: OFIsPinned,
+					Kind:  Equal,
+					Value: true,
+				},
+			},
+			expectedMatchingOrders: storedOrders,
+		},
+		{
+			name: "IsUnfillable = false",
+			filters: []OrderFilter{
+				{
+					Field: OFIsUnfillable,
+					Kind:  Equal,
+					Value: false,
+				},
+			},
+			expectedMatchingOrders: storedOrders,
+		},
+		{
+			name: "IsExpired = false",
+			filters: []OrderFilter{
+				{
+					Field: OFIsExpired,
+					Kind:  Equal,
+					Value: false,
+				},
+			},
+			expectedMatchingOrders: storedOrders,
+		},
 		{
 			name: "MakerAddress = Address1",
 			filters: []OrderFilter{

db/dexie_implementation.go

@@ -188,6 +188,7 @@ func (db *DB) GetOrderStatuses(hashes []common.Hash) (statuses []*StoredOrderSta
 		statuses[i] = &StoredOrderStatus{
 			IsStored:                 jsStatus.Get("isStored").Bool(),
 			IsMarkedRemoved:          jsStatus.Get("isMarkedRemoved").Bool(),
+			IsMarkedUnfillable:       jsStatus.Get("isMarkedUnfillable").Bool(),
 			FillableTakerAssetAmount: fillableAmount,
 		}
 	}

db/dexietypes/dexietypes.go

@@ -157,10 +157,16 @@ type Order struct {
 	IsRemoved                uint8          `json:"isRemoved"`
 	IsPinned                 uint8          `json:"isPinned"`
 	IsNotPinned              uint8          `json:"isNotPinned"` // Used in a compound index in queries related to max expiration time.
+	IsUnfillable             uint8          `json:"isUnfillable"`
+	IsExpired                uint8          `json:"isExpired"`
 	ParsedMakerAssetData     string         `json:"parsedMakerAssetData"`
 	ParsedMakerFeeAssetData  string         `json:"parsedMakerFeeAssetData"`
 	LastValidatedBlockNumber *SortedBigInt  `json:"lastValidatedBlockNumber"`
 	LastValidatedBlockHash   common.Hash    `json:"lastValidatedBlockHash"`
+	KeepCancelled            uint8          `json:"keepCancelled"`
+	KeepExpired              uint8          `json:"keepExpired"`
+	KeepFullyFilled          uint8          `json:"keepFullyFilled"`
+	KeepUnfunded             uint8          `json:"keepUnfunded"`
 }
 
 type Metadata struct {
@@ -196,10 +202,16 @@ func OrderToCommonType(order *Order) *types.OrderWithMetadata {
 		LastUpdated:              order.LastUpdated,
 		IsRemoved:                order.IsRemoved == 1,
 		IsPinned:                 order.IsPinned == 1,
+		IsUnfillable:             order.IsUnfillable == 1,
+		IsExpired:                order.IsExpired == 1,
 		ParsedMakerAssetData:     ParsedAssetDataToCommonType(order.ParsedMakerAssetData),
 		ParsedMakerFeeAssetData:  ParsedAssetDataToCommonType(order.ParsedMakerFeeAssetData),
 		LastValidatedBlockNumber: order.LastValidatedBlockNumber.Int,
 		LastValidatedBlockHash:   order.LastValidatedBlockHash,
+		KeepCancelled:            order.KeepCancelled == 1,
+		KeepExpired:              order.KeepExpired == 1,
+		KeepFullyFilled:          order.KeepFullyFilled == 1,
+		KeepUnfunded:             order.KeepUnfunded == 1,
 	}
 }
 
@@ -231,10 +243,16 @@ func OrderFromCommonType(order *types.OrderWithMetadata) *Order {
 		IsRemoved:                BoolToUint8(order.IsRemoved),
 		IsPinned:                 BoolToUint8(order.IsPinned),
 		IsNotPinned:              BoolToUint8(!order.IsPinned),
+		IsUnfillable:             BoolToUint8(order.IsUnfillable),
+		IsExpired:                BoolToUint8(order.IsExpired),
 		ParsedMakerAssetData:     ParsedAssetDataFromCommonType(order.ParsedMakerAssetData),
 		ParsedMakerFeeAssetData:  ParsedAssetDataFromCommonType(order.ParsedMakerFeeAssetData),
 		LastValidatedBlockNumber: NewSortedBigInt(order.LastValidatedBlockNumber),
 		LastValidatedBlockHash:   order.LastValidatedBlockHash,
+		KeepCancelled:            BoolToUint8(order.KeepCancelled),
+		KeepExpired:              BoolToUint8(order.KeepExpired),
+		KeepFullyFilled:          BoolToUint8(order.KeepFullyFilled),
+		KeepUnfunded:             BoolToUint8(order.KeepUnfunded),
 	}
 }