json test suite

ethcore/Cargo.toml

@@ -27,6 +27,7 @@ ethereum-types = "0.9.0"
 evm = { path = "evm" }
 executive-state = { path = "executive-state" }
 futures = "0.1"
+globset = "0.4.5"
 hash-db = "0.15.0"
 itertools = "0.8.2"
 journaldb = { path = "../util/journaldb" }
@@ -53,6 +54,7 @@ rustc-hex = "2.1.0"
 scopeguard = "1.1.0"
 serde = "1.0"
 serde_derive = "1.0"
+serde_json = "1.0"
 snapshot = { path = "snapshot" }
 spec = { path = "spec" }
 state-db = { path = "state-db" }
@@ -65,6 +67,7 @@ unexpected = { path = "../util/unexpected" }
 using_queue = { path = "../miner/using-queue" }
 verification = { path = "./verification" }
 vm = { path = "vm" }
+walkdir = { version = "2.3.1", optional = true } 
 
 [dev-dependencies]
 account-db = { path = "account-db" }
@@ -82,10 +85,8 @@ kvdb-rocksdb = "0.7.0"
 lazy_static = "1.3"
 machine = { path = "./machine", features = ["test-helpers"] }
 parity-runtime = "0.1.1"
-serde_json = "1.0"
 stats = { path = "../util/stats" }
 pod = { path = "pod" }
-tempfile = "3.1"
 trie-standardmap = "0.15.0"
 
 [features]
@@ -114,7 +115,7 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers", "common-types/test-helpers"]
+json-tests = ["env_logger", "test-helpers", "walkdir", "machine/test-helpers", "common-types/test-helpers"]
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile test helpers

ethcore/res/ethereum/runner/full.json

@@ -0,0 +1,128 @@
+{
+	"chain": [
+		{
+			"path": "res/ethereum/tests/BlockchainTests",
+			"skip": [
+				{
+					"reference": "Issue https://github.com/openethereum/openethereum/issues/11616",
+					"names": [
+						"blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5",
+						"UncleFromFrontierInHomestead_FrontierToHomesteadAt5",
+						"UnclePopulation_FrontierToHomesteadAt5",
+						"blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5",
+						"DaoTransactions_HomesteadToDaoAt5",
+						"DaoTransactions_UncleExtradata_HomesteadToDaoAt5",
+						"DaoTransactions_EmptyTransactionAndForkBlocksAhead_HomesteadToDaoAt5",
+						"DaoTransactions_XBlockm1_HomesteadToDaoAt5",
+						"HomesteadOverrideFrontier_FrontierToHomesteadAt5",
+						"CallContractThatCreateContractBeforeAndAfterSwitchover_FrontierToHomesteadAt5",
+						"ContractCreationFailsOnHomestead_FrontierToHomesteadAt5",
+						"EIP150Transition_HomesteadToEIP150At5"
+					],
+					"paths": []
+				},
+				{
+					"reference": "Tests that takes a lot of time to execute, ignored by geth",
+					"names": [],
+					"paths": [
+						"res/ethereum/tests/BlockchainTests/stTimeConsuming/*"
+					]
+				}
+			]
+		},
+		{
+			"path": "res/ethereum/tests/LegacyTests/Constantinople/BlockchainTests",
+			"skip": [
+				{
+					"reference": "Issue https://github.com/openethereum/openethereum/issues/11616",
+					"names": [
+						"blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5",
+						"UncleFromFrontierInHomestead_FrontierToHomesteadAt5",
+						"UnclePopulation_FrontierToHomesteadAt5",
+						"blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5",
+						"DaoTransactions_HomesteadToDaoAt5",
+						"DaoTransactions_UncleExtradata_HomesteadToDaoAt5",
+						"DaoTransactions_EmptyTransactionAndForkBlocksAhead_HomesteadToDaoAt5",
+						"DaoTransactions_XBlockm1_HomesteadToDaoAt5",
+						"HomesteadOverrideFrontier_FrontierToHomesteadAt5",
+						"CallContractThatCreateContractBeforeAndAfterSwitchover_FrontierToHomesteadAt5",
+						"ContractCreationFailsOnHomestead_FrontierToHomesteadAt5",
+						"EIP150Transition_HomesteadToEIP150At5"
+					],
+					"paths": []
+				},
+				{
+					"reference": "Tests that takes a lot of time to execute, ignored by geth",
+					"names": [],
+					"paths": [
+						"res/ethereum/tests/LegacyTests/Constantinople/BlockchainTests/stTimeConsuming/*"
+					]
+				}
+			]
+		}
+	],
+	"state": [
+		{
+			"path": "res/ethereum/tests/GeneralStateTests",
+			"skip": [
+				{
+					"reference": "Tests that takes a lot of time to execute, ignored by geth",
+					"names": {},
+					"paths": [
+						"res/ethereum/tests/GeneralStateTests/stTimeConsuming/*",
+						"res/ethereum/tests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth.json"
+					]
+				}
+			]
+		},
+		{
+			"path": "res/ethereum/tests/LegacyTests/Constantinople/GeneralStateTests",
+			"skip": [
+				{
+					"reference": "Tests that takes a lot of time to execute, ignored by geth",
+					"names": {},
+					"paths": [
+						"res/ethereum/tests/LegacyTests/GeneralStateTests/Constantinople/stTimeConsuming/*",
+						"res/ethereum/tests/LegacyTests/GeneralStateTests/Constantinople/stStaticCall/static_Call1MB1024Calldepth.json"
+					]
+				}
+			]
+		}
+	],
+	"difficulty": [
+		{
+			"path": [
+				"res/ethereum/tests/BasicTests/difficulty.json",
+				"res/ethereum/tests/BasicTests/difficultyMainNetwork.json"
+			],
+			"chainspec": "Foundation"
+		}
+	],
+	"executive": [
+		{
+			"path": "res/ethereum/tests/VMTests"
+		}
+	],
+	"transaction": [
+		{
+			"path": "res/ethereum/tests/TransactionTests"
+		}
+	],
+	"trie": [
+		{
+			"path": [
+				"res/ethereum/tests/TrieTests/trietest.json",
+				"res/ethereum/tests/TrieTests/trieanyorder.json"
+			],
+			"triespec": "Generic"
+		},
+		{
+			"path": [
+				"res/ethereum/tests/TrieTests/hex_encoded_securetrie_test.json",
+				"res/ethereum/tests/TrieTests/trietest_secureTrie.json",
+				"res/ethereum/tests/TrieTests/trieanyorder_secureTrie.json"
+			],
+			"triespec": "Secure"
+		}
+	]
+}