feat: record dropped spans count and reasons

src/globals.ts

@@ -29,21 +29,29 @@ const warnSpansSizeOnce = runOneTimeWrapper((threshold: number, currentSize: num
     `Lumigo tracer is no longer collecting data on the invocation - maximum size of total spans collected (${threshold} bytes allowed, current size is ${currentSize} bytes)`
   );
 }, {});
+export enum droppedSpanReasons {
+  SPANS_STORED_IN_MEMORY_SIZE_LIMIT = 'SPANS_STORED_IN_MEMORY_SIZE_LIMIT',
+  INVOCATION_MAX_LATENCY_LIMIT = 'INVOCATION_MAX_LATENCY_LIMIT',
+}
 
 export class SpansContainer {
   private static spans: { [id: string]: BasicSpan } = {};
   private static currentSpansSize: number = 0;
   private static totalSpans: number = 0;
+  private static droppedSpansReasons: { [reason: string]: number } = {};
 
   static addSpan(span: BasicSpan): boolean {
-    if (!(span.id in this.spans)) {
+    const newSpan = span.id in this.spans;
+    if (!newSpan) {
       // We call add span also for updating spans with their end part
       this.totalSpans += 1;
     }
     // Memory optimization, take up to 10x maxSize because of smart span selection logic
     const maxSpansSize = getMaxSizeForStoredSpansInMemory();
     if (spanHasErrors(span) || this.currentSpansSize <= maxSpansSize) {
       this.spans[span.id] = span;
+
+      // TODO: If the span isn't new we need to subtract the old span size before adding the new size
       this.currentSpansSize += getJSONBase64Size(span);
       logger.debug('Span created', span);
       return true;
@@ -54,9 +62,31 @@ export class SpansContainer {
       maxSpansSize,
     });
     warnSpansSizeOnce(maxSpansSize, this.currentSpansSize);
+
+    if (newSpan) {
+      SpansContainer.recordDroppedSpan(droppedSpanReasons.SPANS_STORED_IN_MEMORY_SIZE_LIMIT, false);
+    }
+    // TODO: If the span isn't new we need to mark the existing span as partially dropped / truncated somehow
+
     return false;
   }
 
+  static recordDroppedSpan(
+    reason: droppedSpanReasons,
+    incrementTotalSpansCounter: boolean = true,
+    numOfDroppedSpans: number = 1
+  ): void {
+    this.droppedSpansReasons[reason] =
+      (this.droppedSpansReasons[reason.valueOf()] || 0) + numOfDroppedSpans;
+    if (incrementTotalSpansCounter) {
+      this.totalSpans += numOfDroppedSpans;
+    }
+  }
+
+  static getDroppedSpansReasons(): { [reason: string]: number } {
+    return this.droppedSpansReasons;
+  }
+
   static getSpans(): BasicSpan[] {
     return Object.values(this.spans);
   }

src/hooks/baseHttp.ts

@@ -1,4 +1,4 @@
-import { SpansContainer, TracerGlobals } from '../globals';
+import { droppedSpanReasons, SpansContainer, TracerGlobals } from '../globals';
 import {
   getCurrentTransactionId,
   getHttpInfo,
@@ -117,19 +117,23 @@ export class BaseHttp {
   }: httpRequestCreatedParams): HttpRequestTracingConfig | undefined {
     // Gather basic info for creating the HTTP span
     const host = BaseHttp._getHostFromOptionsOrUrl({ options, url });
+
+    if (BaseHttp.isBlacklisted(host) || !isValidAlias()) {
+      return undefined;
+    }
+
+    if (GlobalDurationTimer.isTimePassed()) {
+      SpansContainer.recordDroppedSpan(droppedSpanReasons.INVOCATION_MAX_LATENCY_LIMIT);
+      return undefined;
+    }
+
     const headers = options.headers || {};
     options.headers = headers;
     const addedHeaders = {};
     const { awsRequestId } = TracerGlobals.getHandlerInputs().context;
     const transactionId = getCurrentTransactionId();
     const requestRandomId = getRandomId();
 
-    const shouldIgnoreReq =
-      BaseHttp.isBlacklisted(host) || !isValidAlias() || GlobalDurationTimer.isTimePassed();
-    if (shouldIgnoreReq) {
-      return undefined;
-    }
-
     logger.debug('Starting hook', { host, url, headers });
 
     const returnData: HttpRequestTracingConfig = {

src/reporter.ts

@@ -8,15 +8,16 @@ import {
   isString,
   safeExecute,
   shouldScrubDomain,
-  spanHasErrors,
   shouldTryZip,
+  spanHasErrors,
 } from './utils';
 import * as logger from './logger';
 import { HttpSpansAgent } from './httpSpansAgent';
 import { payloadStringify } from './utils/payloadStringify';
 import { decodeHttpBody, getSpanMetadata, spansPrioritySorter } from './spans/awsSpan';
 import untruncateJson from './tools/untrancateJson';
 import { gzipSync } from 'zlib';
+import { droppedSpanReasons, SpansContainer } from './globals';
 
 export const NUMBER_OF_SPANS_IN_REPORT_OPTIMIZATION = 200;
 export const MAX_SPANS_BULK_SIZE = 200;
@@ -153,6 +154,17 @@ export function getPrioritizedSpans(spans: any[], maxSendBytes: number): any[] {
     }
   }
 
+  const spansDropped = spans.length - Object.keys(spansToSend).length;
+  if (spansDropped > 0) {
+    SpansContainer.recordDroppedSpan(
+      droppedSpanReasons.SPANS_STORED_IN_MEMORY_SIZE_LIMIT,
+      false,
+      spansDropped
+    );
+
+    // TODO: update the end span with the new recorded drops
+  }
+
   return Object.values(spansToSend);
 }
 

src/spans/awsSpan.ts

@@ -288,6 +288,7 @@ export const getEndFunctionSpan = (functionSpan, handlerReturnValue) => {
     event,
     envs,
     totalSpans,
+    droppedSpansReasons: SpansContainer.getDroppedSpansReasons(),
   });
   logger.debug('End span created', newSpan);
   return newSpan;