diff --git a/src/globals.ts b/src/globals.ts index 9371353a2..2ed220fab 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -29,14 +29,20 @@ const warnSpansSizeOnce = runOneTimeWrapper((threshold: number, currentSize: num `Lumigo tracer is no longer collecting data on the invocation - maximum size of total spans collected (${threshold} bytes allowed, current size is ${currentSize} bytes)` ); }, {}); +export enum droppedSpanReasons { + SPANS_STORED_IN_MEMORY_SIZE_LIMIT = 'SPANS_STORED_IN_MEMORY_SIZE_LIMIT', + INVOCATION_MAX_LATENCY_LIMIT = 'INVOCATION_MAX_LATENCY_LIMIT', +} export class SpansContainer { private static spans: { [id: string]: BasicSpan } = {}; private static currentSpansSize: number = 0; private static totalSpans: number = 0; + private static droppedSpansReasons: { [reason: string]: number } = {}; static addSpan(span: BasicSpan): boolean { - if (!(span.id in this.spans)) { + const newSpan = span.id in this.spans; + if (!newSpan) { // We call add span also for updating spans with their end part this.totalSpans += 1; } @@ -44,6 +50,8 @@ export class SpansContainer { const maxSpansSize = getMaxSizeForStoredSpansInMemory(); if (spanHasErrors(span) || this.currentSpansSize <= maxSpansSize) { this.spans[span.id] = span; + + // TODO: If the span isn't new we need to subtract the old span size before adding the new size this.currentSpansSize += getJSONBase64Size(span); logger.debug('Span created', span); return true; @@ -54,9 +62,31 @@ export class SpansContainer { maxSpansSize, }); warnSpansSizeOnce(maxSpansSize, this.currentSpansSize); + + if (newSpan) { + SpansContainer.recordDroppedSpan(droppedSpanReasons.SPANS_STORED_IN_MEMORY_SIZE_LIMIT, false); + } + // TODO: If the span isn't new we need to mark the existing span as partially dropped / truncated somehow + return false; } + static recordDroppedSpan( + reason: droppedSpanReasons, + incrementTotalSpansCounter: boolean = true, + numOfDroppedSpans: number = 1 + ): void { + this.droppedSpansReasons[reason] = + (this.droppedSpansReasons[reason.valueOf()] || 0) + numOfDroppedSpans; + if (incrementTotalSpansCounter) { + this.totalSpans += numOfDroppedSpans; + } + } + + static getDroppedSpansReasons(): { [reason: string]: number } { + return this.droppedSpansReasons; + } + static getSpans(): BasicSpan[] { return Object.values(this.spans); } diff --git a/src/hooks/baseHttp.ts b/src/hooks/baseHttp.ts index c705199fd..dade44c34 100644 --- a/src/hooks/baseHttp.ts +++ b/src/hooks/baseHttp.ts @@ -1,4 +1,4 @@ -import { SpansContainer, TracerGlobals } from '../globals'; +import { droppedSpanReasons, SpansContainer, TracerGlobals } from '../globals'; import { getCurrentTransactionId, getHttpInfo, @@ -117,6 +117,16 @@ export class BaseHttp { }: httpRequestCreatedParams): HttpRequestTracingConfig | undefined { // Gather basic info for creating the HTTP span const host = BaseHttp._getHostFromOptionsOrUrl({ options, url }); + + if (BaseHttp.isBlacklisted(host) || !isValidAlias()) { + return undefined; + } + + if (GlobalDurationTimer.isTimePassed()) { + SpansContainer.recordDroppedSpan(droppedSpanReasons.INVOCATION_MAX_LATENCY_LIMIT); + return undefined; + } + const headers = options.headers || {}; options.headers = headers; const addedHeaders = {}; @@ -124,12 +134,6 @@ export class BaseHttp { const transactionId = getCurrentTransactionId(); const requestRandomId = getRandomId(); - const shouldIgnoreReq = - BaseHttp.isBlacklisted(host) || !isValidAlias() || GlobalDurationTimer.isTimePassed(); - if (shouldIgnoreReq) { - return undefined; - } - logger.debug('Starting hook', { host, url, headers }); const returnData: HttpRequestTracingConfig = { diff --git a/src/reporter.ts b/src/reporter.ts index bd0b514ff..cd21da988 100644 --- a/src/reporter.ts +++ b/src/reporter.ts @@ -8,8 +8,8 @@ import { isString, safeExecute, shouldScrubDomain, - spanHasErrors, shouldTryZip, + spanHasErrors, } from './utils'; import * as logger from './logger'; import { HttpSpansAgent } from './httpSpansAgent'; @@ -17,6 +17,7 @@ import { payloadStringify } from './utils/payloadStringify'; import { decodeHttpBody, getSpanMetadata, spansPrioritySorter } from './spans/awsSpan'; import untruncateJson from './tools/untrancateJson'; import { gzipSync } from 'zlib'; +import { droppedSpanReasons, SpansContainer } from './globals'; export const NUMBER_OF_SPANS_IN_REPORT_OPTIMIZATION = 200; export const MAX_SPANS_BULK_SIZE = 200; @@ -153,6 +154,17 @@ export function getPrioritizedSpans(spans: any[], maxSendBytes: number): any[] { } } + const spansDropped = spans.length - Object.keys(spansToSend).length; + if (spansDropped > 0) { + SpansContainer.recordDroppedSpan( + droppedSpanReasons.SPANS_STORED_IN_MEMORY_SIZE_LIMIT, + false, + spansDropped + ); + + // TODO: update the end span with the new recorded drops + } + return Object.values(spansToSend); } diff --git a/src/spans/awsSpan.ts b/src/spans/awsSpan.ts index f396c615d..b64b8ce2a 100644 --- a/src/spans/awsSpan.ts +++ b/src/spans/awsSpan.ts @@ -288,6 +288,7 @@ export const getEndFunctionSpan = (functionSpan, handlerReturnValue) => { event, envs, totalSpans, + droppedSpansReasons: SpansContainer.getDroppedSpansReasons(), }); logger.debug('End span created', newSpan); return newSpan;