diff --git a/packages/vertica-nodejs/lib/connection.js b/packages/vertica-nodejs/lib/connection.js index 9f6ff2f6..eade0b84 100644 --- a/packages/vertica-nodejs/lib/connection.js +++ b/packages/vertica-nodejs/lib/connection.js @@ -17,6 +17,7 @@ var net = require('net') var fs = require('fs') var EventEmitter = require('events').EventEmitter +const glob = require('glob') const { parse, serialize } = require('v-protocol') @@ -330,19 +331,29 @@ class Connection extends EventEmitter { }) } - sendCopyDataFile(msg) { + sendCopyDataFiles(msg) { const buffer = Buffer.alloc(bufferSize); - const fd = fs.openSync(msg.fileName, 'r'); - let bytesRead = 0; - do { - // read bufferSize bytes from the file into our buffer starting at the current position in the file - bytesRead = fs.readSync(fd, buffer, 0, bufferSize, null); - if (bytesRead > 0) { - // Process the chunk (buffer.slice(0, bytesRead)) here - this.sendCopyData(buffer.subarray(0, bytesRead)) - } - } while (bytesRead > 0); - fs.closeSync(fd); + let expandedFileNames = [] + if (/[*?[\]]/.test(msg.fileName)) { // contains glob pattern + const matchingFiles = glob.sync(msg.fileName) + expandedFileNames = expandedFileNames.concat(matchingFiles) + } else { + expandedFileNames.push(msg.fileName) + } + const uniqueFileNames = [...new Set(expandedFileNames)] // remove duplicates + for (const fileName of uniqueFileNames) { + const fd = fs.openSync(fileName, 'r'); + let bytesRead = 0; + do { + // read bufferSize bytes from the file into our buffer starting at the current position in the file + bytesRead = fs.readSync(fd, buffer, 0, bufferSize, null); + if (bytesRead > 0) { + // Process the chunk (buffer.slice(0, bytesRead)) here + this.sendCopyData(buffer.subarray(0, bytesRead)) + } + } while (bytesRead > 0); + fs.closeSync(fd); + } this.sendEndOfBatchRequest() } diff --git a/packages/vertica-nodejs/lib/query.js b/packages/vertica-nodejs/lib/query.js index 079e0b5a..d4ea01ad 100644 --- a/packages/vertica-nodejs/lib/query.js +++ b/packages/vertica-nodejs/lib/query.js @@ -21,6 +21,7 @@ const utils = require('./utils') const fs = require('fs') const fsPromises = require('fs').promises const stream = require('stream') +const glob = require('glob') class Query extends EventEmitter { constructor(config, values, callback) { @@ -265,11 +266,23 @@ class Query extends EventEmitter { async handleVerifyFiles(msg, connection) { if (msg.numFiles !== 0) { // we are copying from file, not stdin - try { // Check if the data file can be read - await fsPromises.access(msg.files[0], fs.constants.R_OK); - } catch (readInputFileErr) { // Can't open input file for reading, send CopyError - connection.sendCopyError(msg.files[0], 0, '', "Unable to open input file for reading") - return; + let expandedFileNames = [] + for (const fileName of msg.files) { + if (/[*?[\]]/.test(fileName)) { // contains glob pattern + const matchingFiles = glob.sync(fileName) + expandedFileNames = expandedFileNames.concat(matchingFiles) + } else { + expandedFileNames.push(fileName) + } + } + const uniqueFileNames = [...new Set(expandedFileNames)] // remove duplicates + for (const fileName of uniqueFileNames) { + try { // Check if the data file can be read + await fsPromises.access(fileName, fs.constants.R_OK); + } catch (readInputFileErr) { // Can't open input file for reading, send CopyError + connection.sendCopyError(fileName, 0, '', "Unable to open input file for reading") + return; + } } } else { // check to make sure the readableStream is in fact a readableStream if (!(this.copyStream instanceof stream.Readable)) { @@ -317,7 +330,7 @@ class Query extends EventEmitter { } handleLoadFile(msg, connection) { - connection.sendCopyDataFile(msg) + connection.sendCopyDataFiles(msg) } handleWriteFile(msg, connection) { diff --git a/packages/vertica-nodejs/mochatest/integration/client/copy-local-file-tests.js b/packages/vertica-nodejs/mochatest/integration/client/copy-local-file-tests.js index 0ad95cfc..6bbe81bf 100644 --- a/packages/vertica-nodejs/mochatest/integration/client/copy-local-file-tests.js +++ b/packages/vertica-nodejs/mochatest/integration/client/copy-local-file-tests.js @@ -13,8 +13,8 @@ describe('Running Copy From Local File Commands', function () { const badFileName = "copy-bad.dat" const goodFilePath = path.join(process.cwd(), goodFileName); const badFilePath = path.join(process.cwd(), badFileName) - const goodFileContents = "1|'a'\n2|'b'\n3|'c'\n4|'d'\n5|'e'" // 5 correctly formatted rows - const badFileContents = "1|'a'\n'b'|2\n3|'c'\n'd'|4\n5|'e'" // rows 2 and 4 malformed + const goodFileContents = "1|a\n2|b\n3|c\n4|d\n5|e\n" // 5 correctly formatted rows + const badFileContents = "6|f\ng|7\n8|h\ni|9\n10|j\n" // rows 2 and 4 malformed // generate temporary test files, create table before tests begin before((done) => { @@ -66,7 +66,7 @@ describe('Running Copy From Local File Commands', function () { assert.equal(res.rows[0]['Rows Loaded'], 3) // 3 good rows in badFileContents fs.readFile('rejects.txt', 'utf8', (err, data) => { assert.equal(err, undefined) - assert.equal(data, "'b'|2\n'd'|4\n") // rows 2 and 4 are malformed + assert.equal(data, "g|7\ni|9\n") // rows 2 and 4 are malformed }) } finally { fs.unlink('rejects.txt', done) @@ -186,10 +186,6 @@ describe('Running Copy From Local File Commands', function () { }) - it('succeeds using glob patterns', function(done) { - done() - }) - it('succeeds with multiple input files', function(done) { pool.query("COPY copyTable FROM LOCAL 'copy-good.dat', 'copy-bad.dat' RETURNREJECTED", (err, res) => { assert.equal(err, undefined) @@ -198,4 +194,16 @@ describe('Running Copy From Local File Commands', function () { done() }) }) + + it('succeeds using glob patterns', function(done) { + pool.query("COPY copyTable FROM LOCAL 'copy-*.dat' RETURNREJECTED", (err, res) => { + assert.equal(err, undefined) + assert.equal(res.rows[0]['Rows Loaded'], 8) // 5 good rows in goodFileContents + assert.equal(res.getRejectedRows().length, 2) // check the length instead of position in case the order of files loaded changes + pool.query({text: "SELECT num FROM copyTable ORDER BY num ASC", rowMode: 'array'}, (err, res) => { + assert.deepEqual(res.rows, [[1],[2],[3],[4],[5],[6],[8],[10]]) // 7 and 9 malformed. + done() + }) + }) + }) })