From db1c6369352a2ed0896974801fb4f4bce8169fb4 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 8 Oct 2024 14:15:46 +0200 Subject: [PATCH 01/13] feat: adding debug logs for node lambda tracer secret masking --- src/utils/payloadStringify.js | 20 +++++++++++++++++++- src/utils/payloadStringify.test.js | 28 ++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 7e71959f..e91187a6 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -32,12 +32,19 @@ const keyToRegexes = ( backwardCompRegexEnvVarName = LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, regexesEnvVarName = LUMIGO_SECRET_MASKING_REGEX ) => { + logger.debug('Getting key to omit regexes', { + regexesList, + backwardCompRegexEnvVarName, + regexesEnvVarName, + }); const fallbackRegexesList = regexesList; const tryParseEnvVar = (envVarName) => { if (process.env[envVarName]) { + logger.debug('Parsing regexes from environment variable', { envVarName }); return parseJsonFromEnvVar(envVarName, true); } + logger.debug('Environment variable not found', { envVarName }); return null; }; @@ -46,7 +53,10 @@ const keyToRegexes = ( tryParseEnvVar(backwardCompRegexEnvVarName) || tryParseEnvVar(regexesEnvVarName) || regexesList; try { - return regexes.map((x) => new RegExp(x, 'i')); + logger.debug('Parsed regexes', { regexes }); + const regexesExp = regexes.map((x) => new RegExp(x, 'i')); + logger.debug('Parsed regexes expressions', { regexesExp }); + return regexesExp; } catch (e) { invalidMaskingRegexWarning(e); logger.warn('Fallback to default regexes list', { fallbackRegexesList }); @@ -258,8 +268,11 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { + logger.debug('Shallow masking payload by regexes', { payload, regexes }); regexes = regexes || keyToOmitRegexes(); + if (isString(payload)) { + logger.debug('Shallow masking string payload'); return payload; } if (typeof payload !== 'object') { @@ -268,6 +281,7 @@ const shallowMaskByRegex = (payload, regexes) => { } return Object.keys(payload).reduce((acc, key) => { if (keyContainsRegex(regexes, key)) { + logger.debug('Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { acc[key] = payload[key]; @@ -277,6 +291,7 @@ const shallowMaskByRegex = (payload, regexes) => { }; export const shallowMask = (context, payload) => { + logger.debug('Shallow masking payload', { context, payload }); let givenSecretRegexes = null; if (context === 'environment') { givenSecretRegexes = getEnvVarsMaskingRegex(); @@ -295,10 +310,13 @@ export const shallowMask = (context, payload) => { } if (givenSecretRegexes === LUMIGO_SECRET_MASKING_ALL_MAGIC) { + logger.debug('Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); return SCRUBBED_TEXT; } else if (givenSecretRegexes) { + logger.debug('Shallow masking payload with given regexes', { givenSecretRegexes }); try { givenSecretRegexes = JSON.parse(givenSecretRegexes); + logger.debug('Parsed given regexes', { givenSecretRegexes }); givenSecretRegexes = givenSecretRegexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index 3c49f833..a8b7afeb 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -313,7 +313,13 @@ describe('payloadStringify', () => { utils.setDebug(); TracerGlobals.setTracerInputs({}); expect(shallowMask('requestBody', 1)).toEqual(1); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ + + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ { msg: '#LUMIGO# - WARNING - "Failed to mask payload, payload is not an object or string"', obj: '1', @@ -325,11 +331,16 @@ describe('payloadStringify', () => { utils.setDebug(); TracerGlobals.setTracerInputs({}); expect(shallowMask('other', { a: 'b', password: 1234 })).toEqual({ a: 'b', password: '****' }); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ - { + + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ + expect.objectContaining({ msg: '#LUMIGO# - WARNING - "Unknown context for shallowMask"', - obj: '"other"', - }, + }), ]); }); @@ -338,7 +349,12 @@ describe('payloadStringify', () => { process.env[LUMIGO_SECRET_MASKING_REGEX] = '["a(a"]'; expect(shallowMask('requestBody', { a: 'b', aa: 'bla' })).toEqual({ a: 'b', aa: 'bla' }); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ expect.objectContaining({ msg: '#LUMIGO# - WARNING - "Failed to parse the given masking regex"', }), From 62e040b6d49d08ad269f626a3a77791eda844681 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 8 Oct 2024 15:52:38 +0200 Subject: [PATCH 02/13] feat: adding debug logs for node lambda tracer secret masking --- src/utils/payloadStringify.js | 16 +++++++++++++++- src/utils/payloadStringify.test.js | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index e91187a6..5884b14e 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -268,6 +268,17 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { + const byPassMaskingKeys = [ + 'LUMIGO_SECRET_MASKING_REGEX', + 'LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP', + 'LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES', + 'LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_HEADERS', + 'LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_BODIES', + 'LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_HEADERS', + 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT_VARIABLES', + 'LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS', + 'LUMIGO_SECRET_MASKING_EXACT_PATH', + ]; logger.debug('Shallow masking payload by regexes', { payload, regexes }); regexes = regexes || keyToOmitRegexes(); @@ -280,7 +291,10 @@ const shallowMaskByRegex = (payload, regexes) => { return payload; } return Object.keys(payload).reduce((acc, key) => { - if (keyContainsRegex(regexes, key)) { + if (byPassMaskingKeys.includes(key)) { + logger.debug('Bypass masking key', key); + acc[key] = payload[key]; + } else if (keyContainsRegex(regexes, key)) { logger.debug('Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index a8b7afeb..850ffbd5 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -3,6 +3,7 @@ import { LUMIGO_SECRET_MASKING_ALL_MAGIC, LUMIGO_SECRET_MASKING_EXACT_PATH, LUMIGO_SECRET_MASKING_REGEX, + OMITTING_KEYS_REGEXES, LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES, LUMIGO_WHITELIST_KEYS_REGEXES, @@ -298,6 +299,24 @@ describe('payloadStringify', () => { expect(shallowMask('requestBody', { a: 'b', aXy: 'bla' })).toEqual({ a: 'b', aXy: '****' }); }); + test('shallowMask -> requestBody -> regex -> bypass', () => { + process.env[LUMIGO_SECRET_MASKING_REGEX] = OMITTING_KEYS_REGEXES; + + expect( + shallowMask('environment', { + LUMIGO_SECRET_MASKING_REGEX: OMITTING_KEYS_REGEXES, + a: 'b', + secret: 'some secret', + password: 'some password', + }) + ).toEqual({ + LUMIGO_SECRET_MASKING_REGEX: OMITTING_KEYS_REGEXES, + a: 'b', + secret: '****', + password: '****', + }); + }); + test('shallowMask -> requestBody -> fallback', () => { expect(shallowMask('requestBody', { a: 'b', password: 'bla' })).toEqual({ a: 'b', From 2755312e75128e4742af601eefab2e30abed941c Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Wed, 9 Oct 2024 08:45:56 +0200 Subject: [PATCH 03/13] feat: code review updates, --- src/utils.ts | 12 ++++++++++++ src/utils/payloadStringify.js | 16 +++------------- src/utils/payloadStringify.test.js | 13 ++++++------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 8881d38f..89435415 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -50,6 +50,18 @@ export const OMITTING_KEYS_REGEXES = [ 'Authorization', ]; +export const BYPASS_MASKING_KEYS = [ + LUMIGO_SECRET_MASKING_REGEX, + LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, + LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES, + LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_HEADERS, + LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_BODIES, + LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_HEADERS, + LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT, + LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS, + LUMIGO_SECRET_MASKING_EXACT_PATH, +]; + export const LUMIGO_EVENT_KEY = '_lumigo'; export const STEP_FUNCTION_UID_KEY = 'step_function_uid'; export const GET_KEY_DEPTH_ENV_KEY = 'LUMIGO_KEY_DEPTH'; diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 5884b14e..fbd72771 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -17,6 +17,7 @@ import { OMITTING_KEYS_REGEXES, parseJsonFromEnvVar, safeExecute, + BYPASS_MASKING_KEYS, } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; @@ -268,17 +269,6 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { - const byPassMaskingKeys = [ - 'LUMIGO_SECRET_MASKING_REGEX', - 'LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP', - 'LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES', - 'LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_HEADERS', - 'LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_BODIES', - 'LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_HEADERS', - 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT_VARIABLES', - 'LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS', - 'LUMIGO_SECRET_MASKING_EXACT_PATH', - ]; logger.debug('Shallow masking payload by regexes', { payload, regexes }); regexes = regexes || keyToOmitRegexes(); @@ -291,8 +281,8 @@ const shallowMaskByRegex = (payload, regexes) => { return payload; } return Object.keys(payload).reduce((acc, key) => { - if (byPassMaskingKeys.includes(key)) { - logger.debug('Bypass masking key', key); + if (BYPASS_MASKING_KEYS.includes(key)) { + logger.debug('Skipping masking of a Lumigo env-var', key); acc[key] = payload[key]; } else if (keyContainsRegex(regexes, key)) { logger.debug('Shallow masking key', key); diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index 850ffbd5..b5e5f900 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -300,20 +300,19 @@ describe('payloadStringify', () => { }); test('shallowMask -> requestBody -> regex -> bypass', () => { - process.env[LUMIGO_SECRET_MASKING_REGEX] = OMITTING_KEYS_REGEXES; + const regex = '[".*X.*"]'; + process.env[LUMIGO_SECRET_MASKING_REGEX] = regex; expect( shallowMask('environment', { - LUMIGO_SECRET_MASKING_REGEX: OMITTING_KEYS_REGEXES, + LUMIGO_SECRET_MASKING_REGEX: regex, a: 'b', - secret: 'some secret', - password: 'some password', + aXy: 'some secret', }) ).toEqual({ - LUMIGO_SECRET_MASKING_REGEX: OMITTING_KEYS_REGEXES, + LUMIGO_SECRET_MASKING_REGEX: regex, a: 'b', - secret: '****', - password: '****', + aXy: '****', }); }); From cc8f25bdcf1c0fa4e38495cca424ffc21736761c Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Wed, 9 Oct 2024 08:57:53 +0200 Subject: [PATCH 04/13] feat: code review updates, --- src/utils/payloadStringify.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index fbd72771..55aed1b8 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -295,7 +295,7 @@ const shallowMaskByRegex = (payload, regexes) => { }; export const shallowMask = (context, payload) => { - logger.debug('Shallow masking payload', { context, payload }); + logger.debug('Shallow masking payload', { context, payloadKeys: Object.keys(payload) }); let givenSecretRegexes = null; if (context === 'environment') { givenSecretRegexes = getEnvVarsMaskingRegex(); From 3a01a8d20e2b0dfa6b3f49d7f6f76625b3361e32 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Wed, 9 Oct 2024 18:18:37 +0200 Subject: [PATCH 05/13] feat: code review updates, --- src/utils/payloadStringify.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 55aed1b8..228e2ddd 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -54,10 +54,7 @@ const keyToRegexes = ( tryParseEnvVar(backwardCompRegexEnvVarName) || tryParseEnvVar(regexesEnvVarName) || regexesList; try { - logger.debug('Parsed regexes', { regexes }); - const regexesExp = regexes.map((x) => new RegExp(x, 'i')); - logger.debug('Parsed regexes expressions', { regexesExp }); - return regexesExp; + return regexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); logger.warn('Fallback to default regexes list', { fallbackRegexesList }); @@ -269,9 +266,11 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { - logger.debug('Shallow masking payload by regexes', { payload, regexes }); + logger.debug('Shallow masking payload by regexes', { + payloadKeys: Object.keys(payload), + regexes, + }); regexes = regexes || keyToOmitRegexes(); - if (isString(payload)) { logger.debug('Shallow masking string payload'); return payload; From bd20a1a7dc4c094ce8e255c3a5c63c9466379371 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Thu, 10 Oct 2024 15:18:09 +0200 Subject: [PATCH 06/13] fix: test-18, resource_class: large --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c657037..61b4027c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,7 +125,7 @@ jobs: environment: - TZ: Asia/Jerusalem - NODE_OPTIONS: --max_old_space_size=1500 - resource_class: medium+ + resource_class: large working_directory: ~/lumigo-node steps: - run: @@ -150,13 +150,13 @@ jobs: - run: name: check types command: npm run check-types + - run: + name: eslint + command: npm run lint - run: name: test command: npm test no_output_timeout: 15m - - run: - name: eslint - command: npm run lint deploy: <<: *defaults From 65ac698aeb2090d74fa3707c46ec809a370ecad0 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Mon, 14 Oct 2024 17:50:10 +0200 Subject: [PATCH 07/13] fix: less logs, --- src/utils/payloadStringify.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 228e2ddd..4afab04c 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -42,10 +42,8 @@ const keyToRegexes = ( const tryParseEnvVar = (envVarName) => { if (process.env[envVarName]) { - logger.debug('Parsing regexes from environment variable', { envVarName }); return parseJsonFromEnvVar(envVarName, true); } - logger.debug('Environment variable not found', { envVarName }); return null; }; @@ -272,7 +270,6 @@ const shallowMaskByRegex = (payload, regexes) => { }); regexes = regexes || keyToOmitRegexes(); if (isString(payload)) { - logger.debug('Shallow masking string payload'); return payload; } if (typeof payload !== 'object') { @@ -281,10 +278,8 @@ const shallowMaskByRegex = (payload, regexes) => { } return Object.keys(payload).reduce((acc, key) => { if (BYPASS_MASKING_KEYS.includes(key)) { - logger.debug('Skipping masking of a Lumigo env-var', key); acc[key] = payload[key]; } else if (keyContainsRegex(regexes, key)) { - logger.debug('Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { acc[key] = payload[key]; @@ -313,13 +308,10 @@ export const shallowMask = (context, payload) => { } if (givenSecretRegexes === LUMIGO_SECRET_MASKING_ALL_MAGIC) { - logger.debug('Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); return SCRUBBED_TEXT; } else if (givenSecretRegexes) { - logger.debug('Shallow masking payload with given regexes', { givenSecretRegexes }); try { givenSecretRegexes = JSON.parse(givenSecretRegexes); - logger.debug('Parsed given regexes', { givenSecretRegexes }); givenSecretRegexes = givenSecretRegexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); From f91d1f4d070627c1b252a41dccccf5535235e126 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 15 Oct 2024 08:45:21 +0200 Subject: [PATCH 08/13] Revert "fix: less logs," This reverts commit 65ac698aeb2090d74fa3707c46ec809a370ecad0. --- src/utils/payloadStringify.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 4afab04c..228e2ddd 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -42,8 +42,10 @@ const keyToRegexes = ( const tryParseEnvVar = (envVarName) => { if (process.env[envVarName]) { + logger.debug('Parsing regexes from environment variable', { envVarName }); return parseJsonFromEnvVar(envVarName, true); } + logger.debug('Environment variable not found', { envVarName }); return null; }; @@ -270,6 +272,7 @@ const shallowMaskByRegex = (payload, regexes) => { }); regexes = regexes || keyToOmitRegexes(); if (isString(payload)) { + logger.debug('Shallow masking string payload'); return payload; } if (typeof payload !== 'object') { @@ -278,8 +281,10 @@ const shallowMaskByRegex = (payload, regexes) => { } return Object.keys(payload).reduce((acc, key) => { if (BYPASS_MASKING_KEYS.includes(key)) { + logger.debug('Skipping masking of a Lumigo env-var', key); acc[key] = payload[key]; } else if (keyContainsRegex(regexes, key)) { + logger.debug('Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { acc[key] = payload[key]; @@ -308,10 +313,13 @@ export const shallowMask = (context, payload) => { } if (givenSecretRegexes === LUMIGO_SECRET_MASKING_ALL_MAGIC) { + logger.debug('Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); return SCRUBBED_TEXT; } else if (givenSecretRegexes) { + logger.debug('Shallow masking payload with given regexes', { givenSecretRegexes }); try { givenSecretRegexes = JSON.parse(givenSecretRegexes); + logger.debug('Parsed given regexes', { givenSecretRegexes }); givenSecretRegexes = givenSecretRegexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); From 0f98748abb5690e2141a91247edcc60c11862df9 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 15 Oct 2024 09:14:01 +0200 Subject: [PATCH 09/13] feat: lumigo secret env variable --- src/utils.ts | 4 ++++ src/utils/payloadStringify.js | 36 +++++++++++++++++++++--------- src/utils/payloadStringify.test.js | 17 ++++++++++++++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 89435415..0fce3213 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -33,6 +33,8 @@ export const LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS = export const LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT = 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT'; export const LUMIGO_SECRET_MASKING_ALL_MAGIC = 'all'; +export const LUMIGO_SECRET_MASKING_DEBUG = 'LUMIGO_SECRET_MASKING_DEBUG'; + export const LUMIGO_SECRET_MASKING_EXACT_PATH = 'LUMIGO_SECRET_MASKING_EXACT_PATH'; export const LUMIGO_WHITELIST_KEYS_REGEXES = 'LUMIGO_WHITELIST_KEYS_REGEXES'; export const LUMIGO_SUPPORT_LARGE_INVOCATIONS = 'LUMIGO_SUPPORT_LARGE_INVOCATIONS'; @@ -435,6 +437,8 @@ export const setSwitchOff = () => (process.env['LUMIGO_SWITCH_OFF'] = 'TRUE'); export const setDebug = () => (process.env['LUMIGO_DEBUG'] = 'TRUE'); +export const setSecretMaskingDebug = () => (process.env['LUMIGO_SECRET_MASKING_DEBUG'] = 'TRUE'); + export const unsetDebug = () => (process.env['LUMIGO_DEBUG'] = undefined); export const setTimeoutTimerDisabled = () => (process.env[TIMEOUT_ENABLE_FLAG] = 'FALSE'); diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 228e2ddd..d51af968 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -14,6 +14,7 @@ import { LUMIGO_SECRET_MASKING_REGEX, LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, LUMIGO_WHITELIST_KEYS_REGEXES, + LUMIGO_SECRET_MASKING_DEBUG, OMITTING_KEYS_REGEXES, parseJsonFromEnvVar, safeExecute, @@ -33,7 +34,7 @@ const keyToRegexes = ( backwardCompRegexEnvVarName = LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, regexesEnvVarName = LUMIGO_SECRET_MASKING_REGEX ) => { - logger.debug('Getting key to omit regexes', { + logSecretMaskingDebug(logger, 'Getting key to omit regexes', { regexesList, backwardCompRegexEnvVarName, regexesEnvVarName, @@ -42,10 +43,8 @@ const keyToRegexes = ( const tryParseEnvVar = (envVarName) => { if (process.env[envVarName]) { - logger.debug('Parsing regexes from environment variable', { envVarName }); return parseJsonFromEnvVar(envVarName, true); } - logger.debug('Environment variable not found', { envVarName }); return null; }; @@ -170,6 +169,16 @@ function innerPathScrubbing(input, secretPaths, uniquePaths, currentPath) { return input; } +function logSecretMaskingDebug(logger, message, additionalData) { + if (process.env[LUMIGO_SECRET_MASKING_DEBUG]) { + if (additionalData) { + logger.debug(message, additionalData); + } else { + logger.debug(message); + } + } +} + export const payloadStringify = ( payload, maxPayloadSize = getEventEntitySize(), @@ -266,13 +275,13 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { - logger.debug('Shallow masking payload by regexes', { + logSecretMaskingDebug(logger, 'Shallow masking payload by regexes', { payloadKeys: Object.keys(payload), regexes, }); regexes = regexes || keyToOmitRegexes(); if (isString(payload)) { - logger.debug('Shallow masking string payload'); + logSecretMaskingDebug(logger, 'Shallow masking string payload'); return payload; } if (typeof payload !== 'object') { @@ -281,10 +290,10 @@ const shallowMaskByRegex = (payload, regexes) => { } return Object.keys(payload).reduce((acc, key) => { if (BYPASS_MASKING_KEYS.includes(key)) { - logger.debug('Skipping masking of a Lumigo env-var', key); + logSecretMaskingDebug(logger, 'Skipping masking of a Lumigo env-var', key); acc[key] = payload[key]; } else if (keyContainsRegex(regexes, key)) { - logger.debug('Shallow masking key', key); + logSecretMaskingDebug(logger, 'Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { acc[key] = payload[key]; @@ -294,7 +303,10 @@ const shallowMaskByRegex = (payload, regexes) => { }; export const shallowMask = (context, payload) => { - logger.debug('Shallow masking payload', { context, payloadKeys: Object.keys(payload) }); + logSecretMaskingDebug(logger, 'Shallow masking payload', { + context, + payloadKeys: Object.keys(payload), + }); let givenSecretRegexes = null; if (context === 'environment') { givenSecretRegexes = getEnvVarsMaskingRegex(); @@ -313,13 +325,15 @@ export const shallowMask = (context, payload) => { } if (givenSecretRegexes === LUMIGO_SECRET_MASKING_ALL_MAGIC) { - logger.debug('Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); + logSecretMaskingDebug(logger, 'Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); return SCRUBBED_TEXT; } else if (givenSecretRegexes) { - logger.debug('Shallow masking payload with given regexes', { givenSecretRegexes }); + logSecretMaskingDebug(logger, 'Shallow masking payload with given regexes', { + givenSecretRegexes, + }); try { givenSecretRegexes = JSON.parse(givenSecretRegexes); - logger.debug('Parsed given regexes', { givenSecretRegexes }); + logSecretMaskingDebug(logger, 'Parsed given regexes', { givenSecretRegexes }); givenSecretRegexes = givenSecretRegexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index b5e5f900..0ddf59e3 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -382,6 +382,23 @@ describe('payloadStringify', () => { ]); }); + test('shallowMask -> LUMIGO_SECRET_MASKING_DEBUG', () => { + utils.setDebug(); + utils.setSecretMaskingDebug(); + + expect(shallowMask('requestBody', { a: 'b' })).toEqual({ a: 'b' }); + + const debugLogs = ConsoleWritesForTesting.getLogs().filter((log) => log.msg.includes('DEBUG')); + + expect(debugLogs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + msg: '#LUMIGO# - DEBUG - "Shallow masking payload"', + }), + ]) + ); + }); + test.each` envVarValue | event | expectedResults ${['["object.foo"]']} | ${[{ secret: { key: 'value' } }, { object: { foo: 'value' } }]} | ${JSON.stringify([{ secret: '****' }, { object: { foo: '****' } }])} From 765df5d8a8c0b8aa61dede9b661e5c683801b2fe Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 15 Oct 2024 09:25:26 +0200 Subject: [PATCH 10/13] feat: lumigo secret env variable --- src/utils.ts | 2 ++ src/utils/payloadStringify.js | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 0fce3213..bd2cc65a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -353,6 +353,8 @@ export const isWarm = (): boolean => export const isDebug = (): boolean => validateEnvVar(DEBUG_FLAG) || TracerGlobals.getTracerInputs().debug; +export const isSecretMaskingDebug = (): boolean => validateEnvVar(LUMIGO_SECRET_MASKING_DEBUG); + export const isLambdaWrapped = (): boolean => validateEnvVar(WRAPPED_FLAG); export const shouldPropagateW3C = (): boolean => !validateEnvVar(LUMIGO_PROPAGATE_W3C, 'FALSE'); diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index d51af968..61ac8bfb 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -14,11 +14,11 @@ import { LUMIGO_SECRET_MASKING_REGEX, LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, LUMIGO_WHITELIST_KEYS_REGEXES, - LUMIGO_SECRET_MASKING_DEBUG, OMITTING_KEYS_REGEXES, parseJsonFromEnvVar, safeExecute, BYPASS_MASKING_KEYS, + isSecretMaskingDebug, } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; @@ -170,7 +170,7 @@ function innerPathScrubbing(input, secretPaths, uniquePaths, currentPath) { } function logSecretMaskingDebug(logger, message, additionalData) { - if (process.env[LUMIGO_SECRET_MASKING_DEBUG]) { + if (isSecretMaskingDebug()) { if (additionalData) { logger.debug(message, additionalData); } else { From cae0b7da2356dbfd737289b6d603ac042ebb70cf Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 15 Oct 2024 09:27:12 +0200 Subject: [PATCH 11/13] feat: lumigo secret env variable --- src/utils.test.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils.test.js b/src/utils.test.js index 39fda672..f036fd40 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -34,6 +34,7 @@ import { LUMIGO_SUPPORT_LARGE_INVOCATIONS, removeLumigoFromError, removeLumigoFromStacktrace, + LUMIGO_SECRET_MASKING_DEBUG, } from './utils'; describe('utils', () => { @@ -248,6 +249,12 @@ describe('utils', () => { expect(utils.isDebug()).toBe(true); }); + test('isSecretMaskingDebug -> ENV VAR', () => { + expect(utils.isSecretMaskingDebug()).toBe(false); + process.env.LUMIGO_SECRET_MASKING_DEBUG = 'TRUE'; + expect(utils.isSecretMaskingDebug()).toBe(true); + }); + test('isLambdaWrapped', () => { expect(utils.isLambdaWrapped()).toBe(false); process.env.LUMIGO_IS_WRAPPED = 'TRUE'; From 5ef7759cd294abc8dad02e5b904712d207f4b755 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Tue, 15 Oct 2024 17:51:28 +0200 Subject: [PATCH 12/13] feat: lumigo secret env variable revert --- src/utils.test.js | 7 ------- src/utils.ts | 6 ------ src/utils/payloadStringify.js | 11 ++++------- src/utils/payloadStringify.test.js | 17 ----------------- 4 files changed, 4 insertions(+), 37 deletions(-) diff --git a/src/utils.test.js b/src/utils.test.js index f036fd40..39fda672 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -34,7 +34,6 @@ import { LUMIGO_SUPPORT_LARGE_INVOCATIONS, removeLumigoFromError, removeLumigoFromStacktrace, - LUMIGO_SECRET_MASKING_DEBUG, } from './utils'; describe('utils', () => { @@ -249,12 +248,6 @@ describe('utils', () => { expect(utils.isDebug()).toBe(true); }); - test('isSecretMaskingDebug -> ENV VAR', () => { - expect(utils.isSecretMaskingDebug()).toBe(false); - process.env.LUMIGO_SECRET_MASKING_DEBUG = 'TRUE'; - expect(utils.isSecretMaskingDebug()).toBe(true); - }); - test('isLambdaWrapped', () => { expect(utils.isLambdaWrapped()).toBe(false); process.env.LUMIGO_IS_WRAPPED = 'TRUE'; diff --git a/src/utils.ts b/src/utils.ts index bd2cc65a..89435415 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -33,8 +33,6 @@ export const LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS = export const LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT = 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT'; export const LUMIGO_SECRET_MASKING_ALL_MAGIC = 'all'; -export const LUMIGO_SECRET_MASKING_DEBUG = 'LUMIGO_SECRET_MASKING_DEBUG'; - export const LUMIGO_SECRET_MASKING_EXACT_PATH = 'LUMIGO_SECRET_MASKING_EXACT_PATH'; export const LUMIGO_WHITELIST_KEYS_REGEXES = 'LUMIGO_WHITELIST_KEYS_REGEXES'; export const LUMIGO_SUPPORT_LARGE_INVOCATIONS = 'LUMIGO_SUPPORT_LARGE_INVOCATIONS'; @@ -353,8 +351,6 @@ export const isWarm = (): boolean => export const isDebug = (): boolean => validateEnvVar(DEBUG_FLAG) || TracerGlobals.getTracerInputs().debug; -export const isSecretMaskingDebug = (): boolean => validateEnvVar(LUMIGO_SECRET_MASKING_DEBUG); - export const isLambdaWrapped = (): boolean => validateEnvVar(WRAPPED_FLAG); export const shouldPropagateW3C = (): boolean => !validateEnvVar(LUMIGO_PROPAGATE_W3C, 'FALSE'); @@ -439,8 +435,6 @@ export const setSwitchOff = () => (process.env['LUMIGO_SWITCH_OFF'] = 'TRUE'); export const setDebug = () => (process.env['LUMIGO_DEBUG'] = 'TRUE'); -export const setSecretMaskingDebug = () => (process.env['LUMIGO_SECRET_MASKING_DEBUG'] = 'TRUE'); - export const unsetDebug = () => (process.env['LUMIGO_DEBUG'] = undefined); export const setTimeoutTimerDisabled = () => (process.env[TIMEOUT_ENABLE_FLAG] = 'FALSE'); diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 61ac8bfb..b799c333 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -18,7 +18,6 @@ import { parseJsonFromEnvVar, safeExecute, BYPASS_MASKING_KEYS, - isSecretMaskingDebug, } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; @@ -170,12 +169,10 @@ function innerPathScrubbing(input, secretPaths, uniquePaths, currentPath) { } function logSecretMaskingDebug(logger, message, additionalData) { - if (isSecretMaskingDebug()) { - if (additionalData) { - logger.debug(message, additionalData); - } else { - logger.debug(message); - } + if (additionalData) { + logger.debug(message, additionalData); + } else { + logger.debug(message); } } diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index 0ddf59e3..b5e5f900 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -382,23 +382,6 @@ describe('payloadStringify', () => { ]); }); - test('shallowMask -> LUMIGO_SECRET_MASKING_DEBUG', () => { - utils.setDebug(); - utils.setSecretMaskingDebug(); - - expect(shallowMask('requestBody', { a: 'b' })).toEqual({ a: 'b' }); - - const debugLogs = ConsoleWritesForTesting.getLogs().filter((log) => log.msg.includes('DEBUG')); - - expect(debugLogs).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - msg: '#LUMIGO# - DEBUG - "Shallow masking payload"', - }), - ]) - ); - }); - test.each` envVarValue | event | expectedResults ${['["object.foo"]']} | ${[{ secret: { key: 'value' } }, { object: { foo: 'value' } }]} | ${JSON.stringify([{ secret: '****' }, { object: { foo: '****' } }])} From 68a9b2e96c9f22823a458398c7cf3bc4745d2649 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Wed, 16 Oct 2024 16:51:51 +0200 Subject: [PATCH 13/13] feat: lumigo secret env variable --- src/utils.test.js | 7 +++++++ src/utils.ts | 6 ++++++ src/utils/payloadStringify.js | 11 +++++++---- src/utils/payloadStringify.test.js | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/utils.test.js b/src/utils.test.js index 39fda672..f036fd40 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -34,6 +34,7 @@ import { LUMIGO_SUPPORT_LARGE_INVOCATIONS, removeLumigoFromError, removeLumigoFromStacktrace, + LUMIGO_SECRET_MASKING_DEBUG, } from './utils'; describe('utils', () => { @@ -248,6 +249,12 @@ describe('utils', () => { expect(utils.isDebug()).toBe(true); }); + test('isSecretMaskingDebug -> ENV VAR', () => { + expect(utils.isSecretMaskingDebug()).toBe(false); + process.env.LUMIGO_SECRET_MASKING_DEBUG = 'TRUE'; + expect(utils.isSecretMaskingDebug()).toBe(true); + }); + test('isLambdaWrapped', () => { expect(utils.isLambdaWrapped()).toBe(false); process.env.LUMIGO_IS_WRAPPED = 'TRUE'; diff --git a/src/utils.ts b/src/utils.ts index 89435415..bd2cc65a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -33,6 +33,8 @@ export const LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS = export const LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT = 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT'; export const LUMIGO_SECRET_MASKING_ALL_MAGIC = 'all'; +export const LUMIGO_SECRET_MASKING_DEBUG = 'LUMIGO_SECRET_MASKING_DEBUG'; + export const LUMIGO_SECRET_MASKING_EXACT_PATH = 'LUMIGO_SECRET_MASKING_EXACT_PATH'; export const LUMIGO_WHITELIST_KEYS_REGEXES = 'LUMIGO_WHITELIST_KEYS_REGEXES'; export const LUMIGO_SUPPORT_LARGE_INVOCATIONS = 'LUMIGO_SUPPORT_LARGE_INVOCATIONS'; @@ -351,6 +353,8 @@ export const isWarm = (): boolean => export const isDebug = (): boolean => validateEnvVar(DEBUG_FLAG) || TracerGlobals.getTracerInputs().debug; +export const isSecretMaskingDebug = (): boolean => validateEnvVar(LUMIGO_SECRET_MASKING_DEBUG); + export const isLambdaWrapped = (): boolean => validateEnvVar(WRAPPED_FLAG); export const shouldPropagateW3C = (): boolean => !validateEnvVar(LUMIGO_PROPAGATE_W3C, 'FALSE'); @@ -435,6 +439,8 @@ export const setSwitchOff = () => (process.env['LUMIGO_SWITCH_OFF'] = 'TRUE'); export const setDebug = () => (process.env['LUMIGO_DEBUG'] = 'TRUE'); +export const setSecretMaskingDebug = () => (process.env['LUMIGO_SECRET_MASKING_DEBUG'] = 'TRUE'); + export const unsetDebug = () => (process.env['LUMIGO_DEBUG'] = undefined); export const setTimeoutTimerDisabled = () => (process.env[TIMEOUT_ENABLE_FLAG] = 'FALSE'); diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index b799c333..61ac8bfb 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -18,6 +18,7 @@ import { parseJsonFromEnvVar, safeExecute, BYPASS_MASKING_KEYS, + isSecretMaskingDebug, } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; @@ -169,10 +170,12 @@ function innerPathScrubbing(input, secretPaths, uniquePaths, currentPath) { } function logSecretMaskingDebug(logger, message, additionalData) { - if (additionalData) { - logger.debug(message, additionalData); - } else { - logger.debug(message); + if (isSecretMaskingDebug()) { + if (additionalData) { + logger.debug(message, additionalData); + } else { + logger.debug(message); + } } } diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index b5e5f900..0ddf59e3 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -382,6 +382,23 @@ describe('payloadStringify', () => { ]); }); + test('shallowMask -> LUMIGO_SECRET_MASKING_DEBUG', () => { + utils.setDebug(); + utils.setSecretMaskingDebug(); + + expect(shallowMask('requestBody', { a: 'b' })).toEqual({ a: 'b' }); + + const debugLogs = ConsoleWritesForTesting.getLogs().filter((log) => log.msg.includes('DEBUG')); + + expect(debugLogs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + msg: '#LUMIGO# - DEBUG - "Shallow masking payload"', + }), + ]) + ); + }); + test.each` envVarValue | event | expectedResults ${['["object.foo"]']} | ${[{ secret: { key: 'value' } }, { object: { foo: 'value' } }]} | ${JSON.stringify([{ secret: '****' }, { object: { foo: '****' } }])}