diff --git a/.eslintrc.js b/.eslintrc.js index 19ad3ff8e..5c667b5a5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -2,6 +2,9 @@ module.exports = { parser: 'babel-eslint', extends: ['eslint:recommended', 'prettier'], plugins: ['jest'], + rules: { + camelcase: "error" + }, globals: { process: true, console: true, diff --git a/src/parsers/eventParser.js b/src/parsers/eventParser.js index a10364839..2cd114565 100644 --- a/src/parsers/eventParser.js +++ b/src/parsers/eventParser.js @@ -39,6 +39,28 @@ const SNS_KEYS_ORDER = getEnvVarAsList('LUMIGO_SNS_KEYS_ORDER', [ 'MessageId', ]); +const S3_KEYS_ORDER = getEnvVarAsList('LUMIGO_S3_KEYS_ORDER', [ + 'awsRegion', + 'eventTime', + 'eventName', + 'userIdentity', + 'requestParameters', +]); + +const S3_BUCKET_KEYS_ORDER = getEnvVarAsList('LUMIGO_S3_OBJECT_KEYS_ORDER', ['arn']); + +const S3_OBJECT_KEYS_ORDER = getEnvVarAsList('LUMIGO_S3_OBJECT_KEYS_ORDER', ['key', 'size']); + +const CLOUDFRONT_KEYS_ORDER = getEnvVarAsList('LUMIGO_CLOUDFRONT_KEYS_ORDER', ['config']); + +const CLOUDFRONT_REQUEST_KEYS_ORDER = getEnvVarAsList('LUMIGO_CLOUDFRONT_REQUEST_KEYS_ORDER', [ + 'body', + 'clientIp', + 'method', + 'querystring', + 'uri', +]); + export const isApiGwEvent = event => { return ( event != null && @@ -66,73 +88,149 @@ export const isSqsEvent = event => { ); }; +export const isS3Event = event => { + return ( + event != null && + event.Records != null && + event.Records[0] != null && + event.Records[0].eventSource === 'aws:s3' + ); +}; + +export const isCloudfrontEvent = event => { + return ( + event != null && + event.Records != null && + event.Records[0] != null && + event.Records[0].cf != null && + event.Records[0].cf.config != null && + event.Records[0].cf.config.distributionId != null + ); +}; + export const parseApiGwEvent = event => { - const parsed_event = {}; + const parsedEvent = {}; // Add order keys - for (const order_key of API_GW_KEYS_ORDER) { - if (event[order_key] != null) { - parsed_event[order_key] = event[order_key]; + for (const orderKey of API_GW_KEYS_ORDER) { + if (event[orderKey] != null) { + parsedEvent[orderKey] = event[orderKey]; } } // Remove requestContext keys if (event.requestContext != null) { - parsed_event['requestContext'] = {}; - for (const rc_key of Object.keys(event.requestContext)) { - if (API_GW_REQUEST_CONTEXT_FILTER_KEYS.includes(rc_key.toLowerCase())) { - parsed_event['requestContext'][rc_key] = event['requestContext'][rc_key]; + parsedEvent['requestContext'] = {}; + for (const rcKey of Object.keys(event.requestContext)) { + if (API_GW_REQUEST_CONTEXT_FILTER_KEYS.includes(rcKey.toLowerCase())) { + parsedEvent['requestContext'][rcKey] = event['requestContext'][rcKey]; } } } // Remove headers keys if (event.headers != null) { - parsed_event['headers'] = {}; - for (const h_key of Object.keys(event.headers)) { + parsedEvent['headers'] = {}; + for (const hKey of Object.keys(event.headers)) { if ( - API_GW_PREFIX_KEYS_HEADERS_DELETE_KEYS.find(v => h_key.toLowerCase().startsWith(v)) == null + API_GW_PREFIX_KEYS_HEADERS_DELETE_KEYS.find(v => hKey.toLowerCase().startsWith(v)) == null ) { - parsed_event['headers'][h_key] = event['headers'][h_key]; + parsedEvent['headers'][hKey] = event['headers'][hKey]; } } } // Add all other keys for (const key of Object.keys(event)) { if (!API_GW_KEYS_ORDER.includes(key) && !API_GW_KEYS_DELETE_KEYS.includes(key)) { - parsed_event[key] = event[key]; + parsedEvent[key] = event[key]; } } - return parsed_event; + return parsedEvent; }; export const parseSnsEvent = event => { - const new_sns_event = {}; - new_sns_event['Records'] = []; + const newSnsEvent = {}; + newSnsEvent['Records'] = []; // Add order keys for (const rec of event['Records']) { - const new_sns_record_event = {}; + const newSnsRecordEvent = {}; for (const key of SNS_KEYS_ORDER) { if (rec.Sns != null && rec.Sns[key] != null) { - new_sns_record_event[key] = rec.Sns[key]; + newSnsRecordEvent[key] = rec.Sns[key]; } } - new_sns_event['Records'].push({ Sns: new_sns_record_event }); + newSnsEvent['Records'].push({ Sns: newSnsRecordEvent }); } - return new_sns_event; + return newSnsEvent; }; export const parseSqsEvent = event => { - const new_sqs_event = {}; - new_sqs_event['Records'] = []; + const newSqsEvent = {}; + newSqsEvent['Records'] = []; // Add order keys for (const rec of event['Records']) { - const new_sqs_record_event = {}; + const newSqsRecordEvent = {}; for (const key of SQS_KEYS_ORDER) { if (rec[key] != null) { - new_sqs_record_event[key] = rec[key]; + newSqsRecordEvent[key] = rec[key]; } } - new_sqs_event['Records'].push(new_sqs_record_event); + newSqsEvent['Records'].push(newSqsRecordEvent); } - return new_sqs_event; + return newSqsEvent; +}; + +export const parseS3Event = event => { + const newS3Event = {}; + newS3Event['Records'] = []; + // Add order keys + for (const rec of event['Records']) { + const newS3RecordEvent = {}; + for (const key of S3_KEYS_ORDER) { + if (rec.hasOwnProperty(key) != null) { + newS3RecordEvent[key] = rec[key]; + } + } + if (rec.hasOwnProperty('s3')) { + newS3RecordEvent.s3 = {}; + if (rec.s3.hasOwnProperty('bucket')) { + newS3RecordEvent.s3.bucket = {}; + for (const key of S3_BUCKET_KEYS_ORDER) { + newS3RecordEvent.s3.bucket[key] = rec.s3.bucket[key]; + } + } + if (rec.s3.hasOwnProperty('object')) { + newS3RecordEvent.s3.object = {}; + for (const key of S3_OBJECT_KEYS_ORDER) { + newS3RecordEvent.s3.object[key] = rec.s3.object[key]; + } + } + } + newS3Event['Records'].push(newS3RecordEvent); + } + return newS3Event; +}; + +export const parseCloudfrontEvent = event => { + const newCloudfrontEvent = {}; + newCloudfrontEvent['Records'] = []; + // Add order keys + for (const rec of event['Records']) { + const cfRecord = rec['cf'] || {}; + const newCloudfrontRecordEvent = { cf: {} }; + for (const key of CLOUDFRONT_KEYS_ORDER) { + if (cfRecord.hasOwnProperty(key) != null) { + newCloudfrontRecordEvent.cf[key] = cfRecord[key]; + } + } + if (cfRecord.hasOwnProperty('request')) { + newCloudfrontRecordEvent.cf.request = {}; + for (const key of CLOUDFRONT_REQUEST_KEYS_ORDER) { + if (cfRecord.request.hasOwnProperty(key)) { + newCloudfrontRecordEvent.cf.request[key] = cfRecord.request[key]; + } + } + } + newCloudfrontEvent['Records'].push(newCloudfrontRecordEvent); + } + return newCloudfrontEvent; }; export const parseEvent = event => { @@ -146,6 +244,12 @@ export const parseEvent = event => { if (isSqsEvent(event)) { return parseSqsEvent(event); } + if (isS3Event(event)) { + return parseS3Event(event); + } + if (isCloudfrontEvent(event)) { + return parseCloudfrontEvent(event); + } } catch (e) { logger.warn('Failed to parse event', e); } diff --git a/src/parsers/eventParser.test.js b/src/parsers/eventParser.test.js index 386befe35..76f7f3c9b 100644 --- a/src/parsers/eventParser.test.js +++ b/src/parsers/eventParser.test.js @@ -20,7 +20,7 @@ describe('event parser', () => { }); test('api gw v1', () => { - const not_order_api_gw_event = { + const notOrderApiGwEvent = { resource: '/add-user', path: '/add-user', httpMethod: 'POST', @@ -36,6 +36,7 @@ describe('event parser', () => { 'CloudFront-Is-Tablet-Viewer': 'false', 'CloudFront-Viewer-Country': 'IL', 'content-type': 'application/json;charset=UTF-8', + // eslint-disable-next-line camelcase customer_id: 'c_1111', Host: 'aaaa.execute-api.us-west-2.amazonaws.com', origin: 'https://aaa.io', @@ -64,6 +65,7 @@ describe('event parser', () => { 'CloudFront-Is-Tablet-Viewer': ['false'], 'CloudFront-Viewer-Country': ['IL'], 'content-type': ['application/json;charset=UTF-8'], + // eslint-disable-next-line camelcase customer_id: ['c_1111'], Host: ['a.execute-api.us-west-2.amazonaws.com'], origin: ['https://aaa.io'], @@ -95,8 +97,11 @@ describe('event parser', () => { 'custom:customer-name': 'a', 'cognito:username': 'aa', aud: '4lidcnek50hi18996gadaop8j0', + // eslint-disable-next-line camelcase event_id: '9fe80735-f265-41d5-a7ca-04b88c2a4a4c', + // eslint-disable-next-line camelcase token_use: 'id', + // eslint-disable-next-line camelcase auth_time: '1587038744', exp: 'Sun Apr 19 08:06:14 UTC 2020', 'custom:role': 'admin', @@ -137,9 +142,9 @@ describe('event parser', () => { isBase64Encoded: false, }; - const order_api_gw_event = parseEvent(not_order_api_gw_event); + const orderApiGwEvent = parseEvent(notOrderApiGwEvent); - expect(JSON.stringify(order_api_gw_event)).toEqual( + expect(JSON.stringify(orderApiGwEvent)).toEqual( JSON.stringify({ resource: '/add-user', path: '/add-user', @@ -156,8 +161,11 @@ describe('event parser', () => { 'custom:customer-name': 'a', 'cognito:username': 'aa', aud: '4lidcnek50hi18996gadaop8j0', + // eslint-disable-next-line camelcase event_id: '9fe80735-f265-41d5-a7ca-04b88c2a4a4c', + // eslint-disable-next-line camelcase token_use: 'id', + // eslint-disable-next-line camelcase auth_time: '1587038744', exp: 'Sun Apr 19 08:06:14 UTC 2020', 'custom:role': 'admin', @@ -169,6 +177,7 @@ describe('event parser', () => { headers: { Authorization: 'auth', 'content-type': 'application/json;charset=UTF-8', + // eslint-disable-next-line camelcase customer_id: 'c_1111', Host: 'aaaa.execute-api.us-west-2.amazonaws.com', origin: 'https://aaa.io', @@ -183,7 +192,7 @@ describe('event parser', () => { }); test('api gw v2', () => { - const not_order_api_gw_event = { + const notOrderApiGwEvent = { version: '2.0', routeKey: 'ANY /nodejs-apig-function-1G3XMPLZXVXYI', rawPath: '/default/nodejs-apig-function-1G3XMPLZXVXYI', @@ -230,9 +239,9 @@ describe('event parser', () => { isBase64Encoded: true, }; - const order_api_gw_event = parseEvent(not_order_api_gw_event); + const orderApiGwEvent = parseEvent(notOrderApiGwEvent); - expect(JSON.stringify(order_api_gw_event)).toEqual( + expect(JSON.stringify(orderApiGwEvent)).toEqual( JSON.stringify({ version: '2.0', routeKey: 'ANY /nodejs-apig-function-1G3XMPLZXVXYI', @@ -262,7 +271,7 @@ describe('event parser', () => { }); test('sns parse', () => { - const not_order_sns_event = { + const notOrderSnsEvent = { Records: [ { EventVersion: '1.0', @@ -315,9 +324,9 @@ describe('event parser', () => { ], }; - const order_sns_event = parseEvent(not_order_sns_event); + const orderSnsEvent = parseEvent(notOrderSnsEvent); - expect(JSON.stringify(order_sns_event)).toEqual( + expect(JSON.stringify(orderSnsEvent)).toEqual( JSON.stringify({ Records: [ { @@ -346,7 +355,7 @@ describe('event parser', () => { }); test('sqs parse', () => { - const not_order_sqs_event = { + const notOrderSqsEvent = { Records: [ { messageId: '059f36b4-87a3-44ab-83d2-661975830a7d', @@ -383,9 +392,9 @@ describe('event parser', () => { ], }; - const order_sqs_event = parseEvent(not_order_sqs_event); + const orderSqsEvent = parseEvent(notOrderSqsEvent); - expect(JSON.stringify(order_sqs_event)).toEqual( + expect(JSON.stringify(orderSqsEvent)).toEqual( JSON.stringify({ Records: [ { @@ -402,4 +411,144 @@ describe('event parser', () => { }) ); }); + + test('s3 parse', () => { + const notOrderedS3Event = { + Records: [ + { + eventVersion: '2.1', + eventSource: 'aws:s3', + awsRegion: 'us-west-2', + eventTime: '2020-09-24T12:00:12.137Z', + eventName: 'ObjectCreated:Put', + userIdentity: { principalId: 'A2QVTU9T5VMOU3' }, + requestParameters: { sourceIPAddress: '77.127.93.97' }, + responseElements: { + 'x-amz-request-id': '318F33BA8C4CBDC5', + 'x-amz-id-2': + 'VyRyYV/2vjikRUkRoH2WPH6M5WcAjNSGXG8Qtd1uEfbklnTusaDEz/jQPdLQgf2tZLjRuq4MgZFcVFpQJgZLJfiGUoh7IBhU', + }, + s3: { + s3SchemaVersion: '1.0', + configurationId: 'a078ca2d-53a8-45d4-a621-260a439876d8', + bucket: { + name: 'testingbuckets3testing', + ownerIdentity: { principalId: 'A2QVTU9T5VMOU3' }, + arn: 'arn:aws:s3:::testingbuckets3testing', + }, + object: { + key: 'Screen+Shot+2020-05-27+at+12.37.36.png', + size: 61211, + eTag: '714ee5196a5c0a6e6b9019caa7b6e970', + sequencer: '005F6C8A510EE02021', + }, + }, + }, + ], + }; + + const orderedS3Event = parseEvent(notOrderedS3Event); + + expect(JSON.stringify(orderedS3Event)).toEqual( + JSON.stringify({ + Records: [ + { + awsRegion: 'us-west-2', + eventTime: '2020-09-24T12:00:12.137Z', + eventName: 'ObjectCreated:Put', + userIdentity: { principalId: 'A2QVTU9T5VMOU3' }, + requestParameters: { sourceIPAddress: '77.127.93.97' }, + s3: { + bucket: { + arn: 'arn:aws:s3:::testingbuckets3testing', + }, + object: { + key: 'Screen+Shot+2020-05-27+at+12.37.36.png', + size: 61211, + }, + }, + }, + ], + }) + ); + }); + + test('cloudfront parse', () => { + const notOrderedCloudfrontEvent = { + Records: [ + { + cf: { + config: { + distributionDomainName: 'd3f1hyel7d5adt.cloudfront.net', + distributionId: 'E8PDQHVQH1V0Q', + eventType: 'origin-request', + requestId: 'hnql0vH8VDvTTLGwmKn337OH08mMiV5sTPsYGyBqCKgCXPZbfNqYlw==', + }, + request: { + body: { + action: 'read-only', + data: '', + encoding: 'base64', + inputTruncated: false, + }, + clientIp: '176.12.196.206', + headers: { + 'x-forwarded-for': [{ key: 'X-Forwarded-For', value: '176.12.196.206' }], + 'user-agent': [{ key: 'User-Agent', value: 'Amazon CloudFront' }], + via: [ + { + key: 'Via', + value: '1.1 67f7ae71b3a190dab6b84c5ceb7fd5e0.cloudfront.net (CloudFront)', + }, + ], + 'accept-encoding': [{ key: 'Accept-Encoding', value: 'gzip' }], + host: [{ key: 'Host', value: 'my-cloudfront-demo-test.s3.amazonaws.com' }], + }, + method: 'GET', + origin: { + s3: { + authMethod: 'none', + customHeaders: {}, + domainName: 'my-cloudfront-demo-test.s3.amazonaws.com', + path: '', + }, + }, + querystring: '', + uri: '/favicon.ico', + }, + }, + }, + ], + }; + const orderedCloudfrontEvent = parseEvent(notOrderedCloudfrontEvent); + + expect(JSON.stringify(orderedCloudfrontEvent)).toEqual( + JSON.stringify({ + Records: [ + { + cf: { + config: { + distributionDomainName: 'd3f1hyel7d5adt.cloudfront.net', + distributionId: 'E8PDQHVQH1V0Q', + eventType: 'origin-request', + requestId: 'hnql0vH8VDvTTLGwmKn337OH08mMiV5sTPsYGyBqCKgCXPZbfNqYlw==', + }, + request: { + body: { + action: 'read-only', + data: '', + encoding: 'base64', + inputTruncated: false, + }, + clientIp: '176.12.196.206', + method: 'GET', + querystring: '', + uri: '/favicon.ico', + }, + }, + }, + ], + }) + ); + }); }); diff --git a/src/spans/awsSpan.js b/src/spans/awsSpan.js index 844891e5f..04f99fed5 100644 --- a/src/spans/awsSpan.js +++ b/src/spans/awsSpan.js @@ -156,11 +156,11 @@ export const getEndFunctionSpan = (functionSpan, handlerReturnValue) => { const id = removeStartedFromId(functionSpan.id); let error = err ? parseErrorObject(err) : undefined; const ended = new Date().getTime(); - let return_value; + let returnValue; try { - return_value = payloadStringify(data); + returnValue = payloadStringify(data); } catch (e) { - return_value = prune(data.toString(), getEventEntitySize(true)); + returnValue = prune(data.toString(), getEventEntitySize(true)); error = parseErrorObject({ name: 'ReturnValueError', message: `Could not JSON.stringify the return value. This will probably fail the lambda. Original error: ${e && @@ -173,7 +173,8 @@ export const getEndFunctionSpan = (functionSpan, handlerReturnValue) => { id, ended, error, - return_value, + // eslint-disable-next-line camelcase + return_value: returnValue, [EXECUTION_TAGS_KEY]: ExecutionTags.getTags(), event, envs, @@ -321,4 +322,5 @@ export const getHttpSpan = ( }; export const addRttToFunctionSpan = (functionSpan, rtt) => + // eslint-disable-next-line camelcase Object.assign({}, functionSpan, { reporter_rtt: rtt }); diff --git a/src/spans/awsSpan.test.js b/src/spans/awsSpan.test.js index f4cfdf7ca..e3f8d7d5d 100644 --- a/src/spans/awsSpan.test.js +++ b/src/spans/awsSpan.test.js @@ -324,6 +324,7 @@ describe('awsSpan', () => { envs: null, event: null, maxFinishTime: 895093323456, + // eslint-disable-next-line camelcase return_value: '"data man"', [EXECUTION_TAGS_KEY]: [], }; @@ -382,6 +383,7 @@ describe('awsSpan', () => { error: parseErrorObject(err), event: payloadStringify(event), maxFinishTime: 895093323456, + // eslint-disable-next-line camelcase return_value: '', [EXECUTION_TAGS_KEY]: [], }; @@ -520,7 +522,7 @@ describe('awsSpan', () => { expect(awsSpan.getHttpInfo(requestData, responseData)).toEqual(expected); - const scrubbed_expected = { + const scrubbedExpected = { host: 'your.mind.com', request: { body: 'The data is not available', @@ -532,7 +534,7 @@ describe('awsSpan', () => { }; process.env.LUMIGO_DOMAINS_SCRUBBER = '["mind"]'; - expect(awsSpan.getHttpInfo(requestData, responseData)).toEqual(scrubbed_expected); + expect(awsSpan.getHttpInfo(requestData, responseData)).toEqual(scrubbedExpected); }); test('getBasicChildSpan', () => { @@ -907,6 +909,7 @@ describe('awsSpan', () => { test('addRttToFunctionSpan', () => { const functionSpan = { a: 'b' }; const rtt = 1234; + // eslint-disable-next-line camelcase const expected = { reporter_rtt: rtt, ...functionSpan }; expect(awsSpan.addRttToFunctionSpan(functionSpan, rtt)).toEqual(expected); }); diff --git a/src/utils.test.js b/src/utils.test.js index 0d93ae603..1ebed53d0 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -757,22 +757,22 @@ describe('utils', () => { }); test('shouldScrubDomain', () => { - let undefined_url = undefined; - let secrets_manager_url = 'secretsmanager-test.amazonaws.com'; - let google_url = 'http://google.com/'; - let facebook_url = 'http://test.facebook.io/'; - let instagram_url = 'http://test.instagram.io/'; + let undefinedUrl = undefined; + let secretsManagerUrl = 'secretsmanager-test.amazonaws.com'; + let googleUrl = 'http://google.com/'; + let facebookUrl = 'http://test.facebook.io/'; + let instagramUrl = 'http://test.instagram.io/'; - expect(shouldScrubDomain(undefined_url)).toEqual(false); - expect(shouldScrubDomain(secrets_manager_url)).toEqual(true); // checking default scrubbing configuration - expect(shouldScrubDomain(google_url)).toEqual(false); + expect(shouldScrubDomain(undefinedUrl)).toEqual(false); + expect(shouldScrubDomain(secretsManagerUrl)).toEqual(true); // checking default scrubbing configuration + expect(shouldScrubDomain(googleUrl)).toEqual(false); process.env.LUMIGO_DOMAINS_SCRUBBER = '["google"]'; - expect(shouldScrubDomain(google_url)).toEqual(true); + expect(shouldScrubDomain(googleUrl)).toEqual(true); process.env.LUMIGO_DOMAINS_SCRUBBER = '["google", "facebook"]'; - expect(shouldScrubDomain(facebook_url)).toEqual(true); - expect(shouldScrubDomain(instagram_url)).toEqual(false); + expect(shouldScrubDomain(facebookUrl)).toEqual(true); + expect(shouldScrubDomain(instagramUrl)).toEqual(false); }); test('safeExecute run function', () => { diff --git a/src/utils/requireUtils.js b/src/utils/requireUtils.js index c2b37b122..b73617a51 100644 --- a/src/utils/requireUtils.js +++ b/src/utils/requireUtils.js @@ -3,7 +3,7 @@ import * as logger from '../logger'; export const safeRequire = libId => { try { const customReq = - // eslint-disable-next-line no-undef + // eslint-disable-next-line no-undef,camelcase typeof __non_webpack_require__ !== 'undefined' ? __non_webpack_require__ : require; const path = customReq.resolve(libId, { paths: [...process.env.NODE_PATH.split(':'), '/var/task/node_modules/'],