diff --git a/index.d.ts b/index.d.ts index 2293930..284febc 100644 --- a/index.d.ts +++ b/index.d.ts @@ -20,7 +20,7 @@ import { StatObjectResult } from "./StorageResponseInterface"; -export declare type callback = (e?: Error, respBody?: any, respInfo?: any) => void; +export declare type callback = (e?: Error, respBody?: T, respInfo?: any) => void; export declare namespace auth { namespace digest { @@ -1050,6 +1050,11 @@ export declare namespace fop { * 结果是否强制覆盖已有的同名文件 */ force?: boolean; + + /** + * 为 `1` 时开启闲时任务 + */ + type?: number; } class OperationManager { mac: auth.digest.Mac; @@ -1066,14 +1071,44 @@ export declare namespace fop { * @param options * @param callback */ - pfop(bucket: string, key: string, fops: string[], pipeline: string, options: PfopOptions | null, callback: callback): void; + pfop( + bucket: string, + key: string, + fops: string[], + pipeline: string, + options: PfopOptions | null, + callback: callback<{ + persistentId: string + }> + ): void; /** * 查询持久化数据处理进度 - * @param persistentId pfop操作返回的持久化处理ID + * @param persistentId pfop 操作返回的持久化处理ID * @param callback */ - prefop(persistentId: string, callback: callback): void; + prefop( + persistentId: string, + callback: callback<{ + id: string, + pipeline: string, + code: number, + desc: string, + reqid: string, + inputBucket: string, + inputKey: string, + creationDate: string, + type: number, + items: { + cmd: string, + code: number, + desc: string, + returnOld: number, + error?: string, + hash?: string, + }[] + }> + ): void; } } @@ -1744,6 +1779,7 @@ export declare namespace rs { persistentOps?: string; persistentNotifyUrl?: string; persistentPipeline?: string; + persistentType?: string; fsizeLimit?: number; fsizeMin?: number; diff --git a/qiniu/fop.js b/qiniu/fop.js index 0ce5202..a1d35cd 100644 --- a/qiniu/fop.js +++ b/qiniu/fop.js @@ -11,26 +11,42 @@ function OperationManager (mac, config) { this.config = config || new conf.Config(); } -// 发送持久化数据处理请求 -// @param bucket - 空间名称 -// @param key - 文件名称 -// @param fops - 处理指令集合 -// @param pipeline - 处理队列名称 -// @param options - 可选参数 -// notifyURL 回调业务服务器,通知处理结果 -// force 结果是否强制覆盖已有的同名文件 -// @param callbackFunc(err, respBody, respInfo) - 回调函数 -OperationManager.prototype.pfop = function (bucket, key, fops, pipeline, - options, callbackFunc) { +/** + * @typedef {function(Error, any, IncomingMessage)} OperationCallback + */ + +/** + * @param {string} bucket 空间名称 + * @param {string} key 文件名称 + * @param {string[]} fops 处理指令 + * @param {string} pipeline 队列名称 + * @param {object} options 可选参数 + * @param {string} [options.notifyURL] 回调业务服务器,通知处理结果 + * @param {boolean} [options.force] 是否强制覆盖已有的同名文件 + * @param {string} [options.type] 为 `1` 时,开启闲时任务 + * @param {OperationCallback} callbackFunc 回调函数 + */ +OperationManager.prototype.pfop = function ( + bucket, + key, + fops, + pipeline, + options, + callbackFunc +) { options = options || {}; // 必须参数 - var reqParams = { + const reqParams = { bucket: bucket, key: key, - pipeline: pipeline, fops: fops.join(';') }; + // pipeline + if (!pipeline) { + delete reqParams.pipeline; + } + // notifyURL if (options.notifyURL) { reqParams.notifyURL = options.notifyURL; @@ -41,6 +57,11 @@ OperationManager.prototype.pfop = function (bucket, key, fops, pipeline, reqParams.force = 1; } + const persistentType = parseInt(options.type, 10); + if (!isNaN(persistentType)) { + reqParams.type = options.type; + } + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { if (err) { callbackFunc(err, null, null); @@ -51,27 +72,32 @@ OperationManager.prototype.pfop = function (bucket, key, fops, pipeline, }; function pfopReq (mac, config, reqParams, callbackFunc) { - var scheme = config.useHttpsDomain ? 'https://' : 'http://'; - var requestURI = scheme + config.zone.apiHost + '/pfop/'; - var reqBody = querystring.stringify(reqParams); - var auth = util.generateAccessToken(mac, requestURI, reqBody); + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const requestURI = scheme + config.zone.apiHost + '/pfop/'; + const reqBody = querystring.stringify(reqParams); + const auth = util.generateAccessToken(mac, requestURI, reqBody); rpc.postWithForm(requestURI, reqBody, auth, callbackFunc); } -// 查询持久化数据处理进度 -// @param persistentId -// @callbackFunc(err, respBody, respInfo) - 回调函数 -OperationManager.prototype.prefop = function (persistentId, callbackFunc) { - var apiHost = 'api.qiniu.com'; +/** + * 查询持久化数据处理进度 + * @param {string} persistentId + * @param {OperationCallback} callbackFunc 回调函数 + */ +OperationManager.prototype.prefop = function ( + persistentId, + callbackFunc +) { + let apiHost = 'api.qiniu.com'; if (this.config.zone) { apiHost = this.config.zone.apiHost; } - var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; - var requestURI = scheme + apiHost + '/status/get/prefop'; - var reqParams = { + const scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + const requestURI = scheme + apiHost + '/status/get/prefop'; + const reqParams = { id: persistentId }; - var reqBody = querystring.stringify(reqParams); + const reqBody = querystring.stringify(reqParams); rpc.postWithForm(requestURI, reqBody, null, callbackFunc); }; diff --git a/qiniu/storage/form.js b/qiniu/storage/form.js index 972f187..cb7f54e 100644 --- a/qiniu/storage/form.js +++ b/qiniu/storage/form.js @@ -222,7 +222,7 @@ function putReq ( * @param {string | null} key * @param {any} body * @param {PutExtra | null} putExtra - * @param {reqCallback} callbackFunc + * @param {reqCallback} [callbackFunc] * @returns {Promise} */ FormUploader.prototype.put = function ( diff --git a/qiniu/storage/rs.js b/qiniu/storage/rs.js index 8f58d79..b5f7fb7 100644 --- a/qiniu/storage/rs.js +++ b/qiniu/storage/rs.js @@ -1695,6 +1695,7 @@ function _putPolicyBuildInKeys () { * @property {string} [persistentOps] * @property {string} [persistentNotifyUrl] * @property {string} [persistentPipeline] + * @property {string} [persistentType] * @property {number} [fsizeLimit] * @property {number} [fsizeMin] * @property {string} [mimeLimit] diff --git a/test/conftest.js b/test/conftest.js index 9ad2162..bed93ae 100644 --- a/test/conftest.js +++ b/test/conftest.js @@ -12,7 +12,7 @@ function getEnvConfig () { } exports.getEnvConfig = getEnvConfig; -function checkEnvConfigAndExit () { +function checkEnvConfigOkOrExit () { const envConfig = getEnvConfig(); if ( Object.keys(envConfig).some(k => !envConfig[k]) @@ -21,7 +21,7 @@ function checkEnvConfigAndExit () { process.exit(0); } } -exports.checkEnvConfigAndExit = checkEnvConfigAndExit; +exports.checkEnvConfigOrExit = checkEnvConfigOkOrExit; /** * @typedef Param diff --git a/test/fop.test.js b/test/fop.test.js index 4a732eb..2bab8b8 100644 --- a/test/fop.test.js +++ b/test/fop.test.js @@ -1,74 +1,182 @@ -const qiniu = require('../index.js'); +const os = require('os'); +const path = require('path'); +const fs = require('fs'); + const should = require('should'); -const proc = require('process'); -const console = require('console'); - -// eslint-disable-next-line no-undef -before(function (done) { - if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { - console.log('should run command `source test-env.sh` first\n'); - process.exit(0); - } - done(); +const qiniu = require('../index.js'); +const { + checkEnvConfigOrExit, + getEnvConfig, + parametrize, + createRandomFile +} = require('./conftest'); + +// file to upload +const testFilePath = path.join(os.tmpdir(), 'nodejs-sdk-test-fop.bin'); + +before(function () { + checkEnvConfigOrExit(); + return Promise.all([ + createRandomFile(testFilePath, (1 << 20) * 5) + ]); +}); + +after(() => { + return Promise.all( + [ + testFilePath + ].map(p => new Promise(resolve => { + fs.unlink(p, err => { + if (err && err.code !== 'ENOENT') { + console.log(`unlink ${p} failed`, err); + } + resolve(); + }); + })) + ); }); -// eslint-disable-next-line no-undef describe('test start fop', function () { this.timeout(0); - var accessKey = proc.env.QINIU_ACCESS_KEY; - var secretKey = proc.env.QINIU_SECRET_KEY; - var srcBucket = proc.env.QINIU_TEST_BUCKET; - var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); - var config = new qiniu.conf.Config(); + const { + accessKey, + secretKey, + bucketName + } = getEnvConfig(); + const mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + const config = new qiniu.conf.Config(); config.useHttpsDomain = true; - config.zone = qiniu.zone.Zone_z0; - - var persistentId; - - it('test video fop', function (done) { - console.log(srcBucket); - - var pipeline = 'sdktest'; - var srcKey = 'qiniu.mp4'; - var operManager = new qiniu.fop.OperationManager(mac, config); - - // 处理指令集合 - var saveBucket = srcBucket; - var fops = [ - 'avthumb/mp4/s/480x320/vb/150k|saveas/' + qiniu.util.urlsafeBase64Encode( - saveBucket + ':qiniu_480x320.mp4'), - 'vframe/jpg/offset/10|saveas/' + qiniu.util.urlsafeBase64Encode( - saveBucket + - ':qiniu_frame1.jpg') - ]; - - var options = { - notifyURL: 'http://api.example.com/pfop/callback', - force: false - }; - - // 持久化数据处理返回的是任务的persistentId,可以根据这个id查询处理状态 - operManager.pfop(srcBucket, srcKey, fops, pipeline, options, - function (err, respBody, respInfo) { - console.log(respBody, respInfo); - should.not.exist(err); - respBody.should.have.keys('persistentId'); - persistentId = respBody.persistentId; - done(); - }); - }); + config.zone = qiniu.zone.Zone_na0; - it('test video prefop', function (done) { - var operManager = new qiniu.fop.OperationManager(mac, config); - // 查询处理状态 - operManager.prefop(persistentId, - function (err, respBody, respInfo) { - console.log(respBody, respInfo); - should.not.exist(err); - respBody.should.have.keys('id', 'pipeline', 'inputBucket', 'inputKey'); - respBody.should.have.property('id', persistentId); - done(); - }); + let persistentId; + + const testParams = parametrize( + { + name: 'persistentType', + values: [ + undefined, + 0, + 1 + ] + } + ); + + testParams.forEach(function (testParam) { + const { + persistentType + } = testParam; + const msg = `params(${JSON.stringify(testParam)})`; + + it(`test video fop; ${msg}`, function (done) { + let pipeline = 'sdktest'; + const srcKey = 'qiniu.mp4'; + const operationManager = new qiniu.fop.OperationManager(mac, config); + + // 处理指令集合 + const srcBucket = bucketName; + const saveBucket = bucketName; + const fop1 = [ + 'avthumb/mp4/s/480x320/vb/150k|saveas/', + qiniu.util.urlsafeBase64Encode( + `${saveBucket}:qiniu_480x320.mp4` + ) + ].join(''); + const fop2 = [ + 'vframe/jpg/offset/10|saveas/', + qiniu.util.urlsafeBase64Encode( + `${saveBucket}:qiniu_frame1.jpg` + ) + ].join(''); + const fops = [fop1, fop2]; + + const options = { + notifyURL: 'http://api.example.com/pfop/callback', + force: false + }; + + if (persistentType !== undefined) { + options.type = persistentType; + pipeline = null; + } + + // 持久化数据处理返回的是任务的persistentId,可以根据这个id查询处理状态 + operationManager.pfop(srcBucket, srcKey, fops, pipeline, options, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('persistentId'); + persistentId = respBody.persistentId; + done(); + }); + }); + + it(`test video prefop; ${msg}`, function (done) { + const operationManager = new qiniu.fop.OperationManager(mac, config); + // 查询处理状态 + operationManager.prefop(persistentId, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('id', 'pipeline', 'inputBucket', 'inputKey'); + respBody.should.have.property('id', persistentId); + if (persistentType) { + should.equal(respBody.type, persistentType); + } + done(); + }); + }); + + it(`test pfop with upload; ${msg}`, function () { + const formUploader = new qiniu.form_up.FormUploader(config); + const key = 'qiniu-pfop-upload-file'; + const persistentKey = [ + 'qiniu-pfop-by-upload', + 'persistentType', + persistentType + ].join('_'); + + const fop1 = [ + 'avinfo|saveas/', + qiniu.util.urlsafeBase64Encode( + `${bucketName}:${persistentKey}` + ) + ].join(''); + const options = { + scope: bucketName, + persistentOps: [ + fop1 + ].join(';'), + persistentType + }; + const putPolicy = new qiniu.rs.PutPolicy(options); + const uploadToken = putPolicy.uploadToken(mac); + const putExtra = new qiniu.form_up.PutExtra(); + + return formUploader.put(uploadToken, key, testFilePath, putExtra) + .then(({ data }) => { + data.should.have.keys('key', 'persistentId'); + + return new Promise((resolve, reject) => { + new qiniu.fop.OperationManager(mac, config) + .prefop( + data.persistentId, + function (err, respBody, respInfo) { + if (err) { + reject(err); + return; + } + resolve({ data: respBody, resp: respInfo }); + } + ); + }); + }) + .then(({ data }) => { + data.should.have.keys('creationDate'); + if (persistentType) { + should.equal(data.type, persistentType); + } + }); + }); }); }); diff --git a/test/form_up.test.js b/test/form_up.test.js index 9eec275..a2d1e7f 100644 --- a/test/form_up.test.js +++ b/test/form_up.test.js @@ -14,7 +14,7 @@ const { const { getEnvConfig, - checkEnvConfigAndExit, + checkEnvConfigOrExit, createRandomFile, doAndWrapResultPromises } = require('./conftest'); @@ -24,7 +24,7 @@ const testFilePath1 = path.join(os.tmpdir(), 'nodejs-sdk-test-1.bin'); const testFilePath2 = path.join(os.tmpdir(), 'nodejs-sdk-test-2.bin'); before(function () { - checkEnvConfigAndExit(); + checkEnvConfigOrExit(); return Promise.all([ createRandomFile(testFilePath1, (1 << 20) * 10), diff --git a/test/resume_up.test.js b/test/resume_up.test.js index dae78f2..15f6b17 100644 --- a/test/resume_up.test.js +++ b/test/resume_up.test.js @@ -16,7 +16,7 @@ const { const { getEnvConfig, - checkEnvConfigAndExit, + checkEnvConfigOrExit, createRandomFile, createRandomStreamAndMD5, doAndWrapResultPromises, @@ -67,7 +67,7 @@ function getRemoteObjectHeadersAndMD5 (url) { } before(function () { - checkEnvConfigAndExit(); + checkEnvConfigOrExit(); return Promise.all([ createRandomFile(testFilePath, (1 << 20) * 10) diff --git a/test/rs.test.js b/test/rs.test.js index f3cf312..e8ee2cf 100644 --- a/test/rs.test.js +++ b/test/rs.test.js @@ -6,12 +6,12 @@ const qiniu = require('../index.js'); const { getEnvConfig, - checkEnvConfigAndExit, + checkEnvConfigOrExit, doAndWrapResultPromises } = require('./conftest'); before(function () { - checkEnvConfigAndExit(); + checkEnvConfigOrExit(); }); describe('test start bucket manager', function () {