diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/lib/cli_out.js b/lib/cli_out.js new file mode 100644 index 0000000..f6c4837 --- /dev/null +++ b/lib/cli_out.js @@ -0,0 +1,39 @@ +'use strict'; + +var clc = require('cli-color-tty')(true); +var path = require('path'); +var through = require('through2'); + +// Takes the output of the test and test_summarizer streams +// and emits it to stdout and stderr (as appropriate) before +// passing it on to the next stream in the pipeline. +var first = true; +module.exports = function () { + var lastDirectory; + + return through.obj(function (file, enc, cb) { + this.push(file); + + if (!/\.(stderr|stdout)$/.test(file.path)) { + return cb(); + } + + var out = console.log; + + if (/\.stderr$/.test(file.path)) { + out = console.error; + } + + if (first) { + out(); + first = false; + } + + if (lastDirectory !== path.dirname(file.path)) { + lastDirectory = path.dirname(file.path); + out(clc.bold(lastDirectory)); + } + out(' ' + String(file.contents)); + cb(); + }); +}; diff --git a/lib/test.js b/lib/test.js new file mode 100644 index 0000000..ab400f1 --- /dev/null +++ b/lib/test.js @@ -0,0 +1,167 @@ +'use strict'; + +var _ = require('lodash'); +var BigNumber = require('bignumber.js'); +var clc = require('cli-color-tty')(true); +var Contract = require('../contract'); +var deasync = require('deasync'); +var File = require('vinyl'); +var LogTranslator = require('../logtranslator'); +var path = require('path'); +var fs = require('fs'); +var through = require('through2'); +var VMTest = require('../vmtest'); + +// This stream takes the output of either the build stream or pipeline (they +// produce the same output) and returns a stream of files containing the output +// of running each `Test` contract. A special, non-standard `error` flag is set +// on File objects representing failed tests. This allows the `cli_out` stream +// to push the content of those files to `stderr` instead of `stdout`. + +function runTests (stream, className, vmTest, logTranslator) { + var testCount = vmTest.testCount(); + var remaining = testCount; + var deployFailure = false; + + function testResultHandler (err, result) { + if (deployFailure || stream.isPaused()) return; + + if (err) { + stream.push(new File({ + path: path.join(className, + 'Deployment failure.stderr'), + contents: new Buffer(String(err)) + })); + deployFailure = true; + return; + } + + var color = clc.green; + + if (result.failed) { + color = clc.red; + } + + // TODO: Clean this up. We want it to be + // easy to have special log formatting for + // particular types of events, and this is + // a discreet logical chunk that belongs in + // its own function or class somewhere. + var output = result.title + '\n'; + var logPrefix = ' | '; + var report = ''; + for (let entry of result.logs) { + if (entry.event === '__startBlockE') { + report += '```{' + entry.args.what + '}\n'; + } else if (entry.event === '__stopBlockE') { + report += '```\n'; + } else if (entry.event.indexOf('log_id_') > -1 && LogTranslator.logs[entry.event].type === 'doc') { + report += LogTranslator.format(entry) + '\n'; + } else if (entry.event.indexOf('_named_') > -1) { + var key = toAscii(entry.args.key) + ': '; + var val = entry.args.decimals + ? toDecimal(entry.args.val, entry.args.decimals) + : entry.args.val; + output += logPrefix + key + val + '\n'; + } else if (entry.event.indexOf('log_id_') > -1) { + output += ' ' + LogTranslator.format(entry) + '\n'; + } else if (entry.event === 'log_bytes32') { + output += logPrefix + toAscii(entry.args.val) + '\n'; + } else { + output += logPrefix + entry.event + '\n'; + + for (let arg of _.pairs(entry.args)) { + output += logPrefix + ' ' + + arg[0] + ': ' + arg[1] + '\n'; + } + } + } + output += ' ' + color(result.message) + '\n'; + // TODO refactor to write report stream file + if (result.reporterPath) { + fs.appendFileSync(result.reporterPath, report); + } + + var file = new File({ + path: path.join( + className, + result.title + (result.failed ? '.stderr' : '.stdout')), + contents: new Buffer(output) + }); + stream.push(file); + + remaining = remaining - 1; + } + + // Run all the tests in parallel. + for (var i = 0; i < testCount; i++) { + vmTest.runTest(i, testResultHandler); + } + + // Wait until all the tests have been run. + deasync.loopWhile(() => remaining !== 0 && !deployFailure); +} + +module.exports = function (opts) { + return through.obj(function (file, enc, cb) { + var classes = JSON.parse(String(file.contents)); + + // Skip if Test contract isn't found + if (!('Test' in classes)) return cb(); + + // Load the Test contract + try { + var testContract = new Contract(classes['Test']); + } catch (err) { + return cb(err); + } + + // TODO - export this to pipeline setup as different streams have to be able to communicate with the chain + var web3 = opts.web3; + + // var libraryAddressMap = {}; + // const DEFAULT_GAS = 900000000; // 900 million + var className; + + for (className in classes) { + // Filter classNames if a filter is present if a filter is present + if (opts.nameFilter && !opts.nameFilter.test(className)) { + continue; + } + + try { + var contract = new Contract(classes[className]); + } catch (err) { + return cb(err); + } + + // way to determine if the class is a test, + // iff it has implemented the Test interface + if (_.intersection(contract.signatures, testContract.signatures) + .length !== testContract.signatures.length) { + continue; + } + let translator = opts.logTranslator || new LogTranslator(contract.abi); + var vmTest = opts.vmTest || new VMTest(web3, contract, translator); + let stream = opts.stream || this; + runTests(stream, className, vmTest, translator); + } + vmTest.stop(); + cb(); + }); +}; + +function toAscii (hex) { + hex = hex.replace(/^0x/, ''); + var result = ''; + for (var i = 0; i < hex.length - 1; i += 2) { + result += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); + } + return result.replace(/\0/g, ''); +} + +function toDecimal (value, decimals) { + return new BigNumber(value) + .div(new BigNumber(10).pow(decimals)) + .toFixed(decimals); +} diff --git a/lib/test_summarizer.js b/lib/test_summarizer.js new file mode 100644 index 0000000..fb4b4f1 --- /dev/null +++ b/lib/test_summarizer.js @@ -0,0 +1,51 @@ +'use strict'; + +var clc = require('cli-color-tty')(true); +var File = require('vinyl'); +var path = require('path'); +var through = require('through2'); + +// Takes the output of the test stream and summarizes the test results. +// Outputs one file object if all the tests passed, and two if some +// tests failed. One file object has a string describing the overall +// test status, and the other has a list of all failing tests. +module.exports = function () { + var totalTests = 0; + var failingTests = []; + + return through.obj(function (file, enc, cb) { + if (/\.stderr$/.test(file.path)) { + failingTests.push(file); + } + totalTests += 1; + cb(); + }, function (cb) { + var ext = '.stdout'; + var output = clc.green('Passed all ' + totalTests + ' tests!'); + + if (failingTests.length > 0) { + ext = '.stderr'; + output = clc.red( + 'Failed ' + failingTests.length + + ' out of ' + totalTests + ' tests.'); + + var failedOutput = ''; + for (let failingTest of failingTests) { + failedOutput += clc.red(path.dirname(failingTest.path) + + ': ' + path.basename(failingTest.path, ext)) + '\n '; + } + + this.push(new File({ + path: path.join('Failing Tests', 'summary' + ext), + contents: new Buffer(failedOutput) + })); + } + + this.push(new File({ + path: path.join('Summary', 'summary' + ext), + contents: new Buffer(output) + })); + + cb(); + }); +}; diff --git a/lib/testpipeline.js b/lib/testpipeline.js index fc5065f..5cee901 100644 --- a/lib/testpipeline.js +++ b/lib/testpipeline.js @@ -1,5 +1,9 @@ "use strict"; +var test = require('./test.js'); +var cli_out = require('./cli_out.js'); +var test_summarizer = require('./test_summarizer.js'); + var Web3Interface = require('dapple-core/web3Interface.js'); var TestPipeline = function (opts) { @@ -14,10 +18,10 @@ var TestPipeline = function (opts) { return Combine( // streams.linkLibraries(opts), - streams.test(opts), - streams.cli_out(), - streams.test_summarizer(), - streams.cli_out() + test(opts), + cli_out(), + test_summarizer(), + cli_out() ); }; diff --git a/lib/vmtest.js b/lib/vmtest.js new file mode 100644 index 0000000..c06db53 --- /dev/null +++ b/lib/vmtest.js @@ -0,0 +1,345 @@ +'use strict'; + +var _ = require('lodash'); +var LogTranslator = require('./logtranslator'); +var utils = require('./utils'); +var fs = require('fs'); +var constants = require('../lib/constants'); + +// DEFAULT_GAS and DEFAULT_ENDOWMENT are arbitrary. +const DEFAULT_GAS = 900000000; // 900 million +const DEFAULT_ENDOWMENT = 10000000; // 10 million +const DEFAULT_FAIL_FUNCTION_NAMES = ['testThrow', 'testFail', 'testError']; + +// TODO use chain manager +// TODO Use transaction manager. Retrying transactions like we do at present +// works, but it's kludgy and unmaintainable. There needs to be a clean +// abstraction around it because we're going to need to re-use those techniques +// elsewhere. + +module.exports = class VMTest { + // Takes a web3 instance, a Contract object (from ./contract.js), + // and a LogTranslator (from ./logTranslator.js). Will later use + // these values to run the tests defined in the given contract. + constructor (web3, contract, logTranslator) { + this.web3 = web3; + this.contract = contract; + this.logTranslator = logTranslator; + this.reporterPath = false; + + if (!logTranslator) { + this.logTranslator = new LogTranslator(contract.abi); + } + + this.tests = []; + for (var item of this.contract.abi) { + if (item.name && item.name.indexOf('test') === 0) { + this.tests.push(item.name); + } + } + } + + static writeTestTemplate (className) { + var template = _.template(constants.SOL_CONTRACT_TEMPLATE()); + var contract = template({ + className: className + }); + + fs.writeFileSync(utils.classToFilename(className), contract); + + var testClassName = className + 'Test'; + var testTemplate = _.template(constants.SOL_CONTRACT_TEST_TEMPLATE()); + var test = testTemplate({ + className: className, + testClassName: testClassName + }); + + fs.writeFileSync(utils.classToFilename(testClassName), test); + } + + static testResult (failed, title, message, logs, reporterPath) { + // TODO: Replace with proper JSON schema validation. + if (typeof failed === 'undefined') { + throw new Error('testResult requires a boolean as the 1st argument.'); + } + + if (typeof title === 'undefined') { + throw new Error('testResult requires a test name as the 2nd argument.'); + } + + if (typeof message === 'undefined') { + message = failed ? 'Failed!' : 'Passed!'; + } + + if (typeof logs === 'undefined') { + logs = []; + } + + return { + title: title, + message: message, + logs: logs, + failed: failed, + reporterPath: reporterPath + }; + } + + // **TODO**: Chain snapshotting + // Takes the array index of a test in `this.tests` and runs it. + // Passes the result to the callback function once it's done. + runTest (testIndex, cb) { + var that = this; + + if (typeof that.web3.eth.defaultAccount === 'undefined') { + that.web3.eth.defaultAccount = that.web3.eth.accounts[0]; + } + + cb = utils.optionalCallback(cb); + + var contractClass = that.web3.eth.contract(that.contract.abi); + + var runTestOn = function (contract) { + that.runInstanceTestByName(contract, that.tests[testIndex], cb); + }; + + var setUpHandlerFor = function (contract) { + return function (err, txHash) { + if (err) { + return cb(err); + } + + that.web3.eth.getTransactionReceipt(txHash, function (err, receipt) { + if (receipt === null && err === null) { + err = 'setUp failed - exception thrown'; + } + + if (err) { + cb(err, VMTest.testResult(true, 'setUp failed', err)); + return; + } + + if (!that.reporterPath) { + var reporterLog = that.logTranslator.translateAll( + receipt.logs.filter(l => l.topics.length > 0 && + l.topics[0] === '0x2ef9ea3c32090ed6e07dce1537a1ff386ee09eb5e752f0387a46c9a90bd31642')); + if (reporterLog.length > 0) { + that.reporterPath = reporterLog[0].args.where; + fs.writeFileSync(that.reporterPath, ''); // clear file + } + } + + runTestOn(contract); + }); + }; + }; + + var getCodeHandlerFor = function (address) { + return function (err, code) { + if (err) { + return cb(err); + } + + if (code === '0x') { + return cb('Contract failed to deploy.'); + } + + var contract = contractClass.at(address); + + if (contract.setUp !== undefined) { + contract.setUp(setUpHandlerFor(contract)); + } else { + runTestOn(contract); + } + }; + }; + + var deployHandler = function (err, receipt) { + if (err) { + return cb(err); + } + that.web3.eth.getCode(receipt.contractAddress, + getCodeHandlerFor(receipt.contractAddress)); + }; + + that.deploy(deployHandler); + } + + testCount () { + return this.tests.length; + } + + stop (cb) { + this.web3.currentProvider.stop(cb); + } + + // Deploys the VMTest's contract to its blockchain. + deploy (cb) { + var that = this; + + function createTxCheck (callback) { + let txCheck = function (err, txHash) { + if (err) { + return callback(err); + } + + that.web3.eth.getTransactionReceipt(txHash, function (err, res) { + if (!err && !res) { + setTimeout(txCheck.bind(this, null, txHash), 2000); + return; + } + callback(err, res); + }); + }; + return txCheck; + } + + that.web3.eth.sendTransaction({ + from: that.web3.eth.defaultAccount, + data: '0x' + this.contract.bytecode, + gas: DEFAULT_GAS, + gasLimit: DEFAULT_GAS + + }, createTxCheck(function (err, receipt) { + if (err) { + return cb(new Error('Contract deployment error: ' + err)); + } + that.web3.eth.sendTransaction({ + from: that.web3.eth.defaultAccount, + to: receipt.contractAddress, + gas: DEFAULT_GAS, + gasLimit: DEFAULT_GAS, + value: DEFAULT_ENDOWMENT + }, createTxCheck(function (err) { + if (err) { + return cb(new Error('Contract endowment error: ' + err)); + } + cb(err, receipt); + })); + })); + } + + // Runs the given test function on the given contract instance. + // Passes back an object representing the results of the test + // to the callback function. + // TODO: Break up some of this nesting. It's really intense. + runInstanceTestByName (contractInstance, testFunction, cb) { + var that = this; + + // returns true, if the function is expecting an error (throw) + var expectingAnError = new RegExp( + DEFAULT_FAIL_FUNCTION_NAMES + .map(s => `^${s}`) + .join('|')) + .test(testFunction); + + var testString = testFunction.replace(/([A-Z])/g, ' $1') + .replace(/_/g, ' ') + .toLowerCase(); + + function captureTestResults (err, txHash) { + if (err) { + if (expectingAnError) { + cb(null, VMTest.testResult(false, testString)); + } else { + cb(null, VMTest.testResult(true, testString, err)); + } + return; + } + + that.web3.eth.getTransactionReceipt(txHash, function (err, receipt) { + if (err) { + return cb(err, txHash); + } + + contractInstance.failed(function (err, failed) { + var message, logs; + if (receipt === null) { + failed = !expectingAnError; + logs = []; + message = failed ? 'test failed - exception thrown' : 'Passed!'; + } else { + failed = failed || Boolean(err); + + logs = that.logTranslator.translateAll(receipt.logs); + + var eventListeners = logs.filter(log => log.event === 'eventListener'); + logs = logs.filter(log => log.event !== 'eventListener'); + + // if expect an event + if (Array.isArray(eventListeners) && eventListeners.length > 0) { + // event must be triggered + + // each event + eventListeners.forEach(expectedEvent => { + if (expectedEvent.args.exact) { + // prepare expected logs + var expected = logs + // Get all logs which should be thrown + .filter(log => log.address === expectedEvent.address) + // make sure the order is right + .sort((a, b) => a.logIndex < b.logIndex); + + // prepare observed logs + var observed = logs + // Get all logs which should be thrown + .filter(log => log.address === expectedEvent.args._target) + // make sure the order is right + .sort((a, b) => a.logIndex < b.logIndex); + + // Number of events has to be the same + if (observed.length !== expected.length) { + failed = true; + } else { + // Events has to be exactly the same + for (var i = 0; i < observed.length; i++) { + // events has to be the same and in correct order + failed = failed || observed[i].event !== expected[i].event; + if (failed) break; + + // all args has to be the same + _.each(observed[i].args, (value, key) => { + failed = failed || expected[i].args[key] !== value; + }); + } + } + + // generate human readable error output + if (failed) { + // + // format events output + var format = function (es) { + es.map((e, i) => { + let args = _.map(e.args, (v, k) => `${k}=${v}`).join(' '); + return `${i}. ${e.event} - ${args}`; + }).join('\n'); + }; + + // format message + message += 'The expected events do not match with the ' + + `observed:\nexpected: ${format(expected)}\n` + + `observed:${format(observed)}`; + } + } + }); + } + + // failed' := not (failed iff expected) + // + // failed and unexpected or + // successful and expected + failed = (failed && !expectingAnError) || + (!failed && expectingAnError); + message = err || (failed ? 'Failed!' : 'Passed!'); + } + + cb(err, VMTest.testResult(failed, testString, message, logs, that.reporterPath)); + }); + }); + } + + contractInstance[testFunction]({ + from: that.web3.eth.defaultAccount, + gas: DEFAULT_GAS + }, captureTestResults); + } +}; diff --git a/package.json b/package.json index a03d899..94466fa 100644 --- a/package.json +++ b/package.json @@ -23,5 +23,13 @@ "bugs": { "url": "https://github.com/mhhf/dapple-test/issues" }, - "homepage": "https://github.com/mhhf/dapple-test#readme" + "homepage": "https://github.com/mhhf/dapple-test#readme", + "dependencies": { + "bignumber.js": "^2.4.0", + "cli-color-tty": "^2.0.0", + "deasync": "^0.1.7", + "lodash": "^4.15.0", + "through2": "^2.0.1", + "vinyl": "^1.2.0" + } }