add direct metrics pushed to datadog

api/package.json

@@ -36,6 +36,7 @@
     "bcrypt": "^5.0.1",
     "cron-parser": "^4.9.0",
     "dayjs": "^1.11.5",
+    "datadog-metrics": "^0.12.0",
     "debug": "^4.3.4",
     "dotenv": "^16.0.1",
     "fast-levenshtein": "^3.0.0",

api/sample.env

@@ -40,6 +40,14 @@ REDIS_URL=redis://localhost:6379
 # default: none
 # sample (everyday at 06:30 UTC): CACHE_RELOAD_TIME=30 6 * * *
 
+# =====
+# DATADOG
+# =====
+
+# Datadog access context
+#
+DATADOG_API_KEY=nokey
+
 # =========
 # DATABASES
 # =========

api/server.js

@@ -1,5 +1,6 @@
 import Oppsy from '@1024pix/oppsy';
 import Hapi from '@hapi/hapi';
+import metrics from 'datadog-metrics';
 import { parse } from 'neoqs';
 
 import { setupErrorHandling } from './config/server-setup-error-handling.js';
@@ -35,12 +36,15 @@ import { config } from './src/shared/config.js';
 import { monitoringTools } from './src/shared/infrastructure/monitoring-tools.js';
 import { plugins } from './src/shared/infrastructure/plugins/index.js';
 import { deserializer } from './src/shared/infrastructure/serializers/jsonapi/deserializer.js';
+import { logger } from './src/shared/infrastructure/utils/logger.js';
 // bounded context migration
 import { sharedRoutes } from './src/shared/routes.js';
 import { swaggers } from './src/shared/swaggers.js';
 import { handleFailAction } from './src/shared/validate.js';
 import { teamRoutes } from './src/team/application/routes.js';
 
+metrics.init({ host: 'myhost', prefix: 'myapp.' });
+
 const certificationRoutes = [
   attachTargetProfileRoutes,
   certificationConfigurationRoutes,
@@ -71,6 +75,21 @@ const createServer = async () => {
 
   if (logOpsMetrics) await enableOpsMetrics(server);
 
+  // initialisation of Datadog link for metrics publication
+  if (config.environment !== 'development' || config.featureToggles.isDirectMetricsEnabled) {
+    logger.info('Metric initialisation : linked to Datadog');
+    metrics.init({
+      host: config.infra.containerName,
+      prefix: 'tests.',
+      defaultTags: [`service:${config.infra.appName}`],
+    });
+  } else {
+    logger.info('Metric initialisation : no reporter => no metrics sent');
+    metrics.init({ reporter: metrics.NullReporter() });
+  }
+
+  if (logOpsMetrics) await enableOpsMetrics(server);
+
   setupErrorHandling(server);
 
   setupAuthentication(server);
@@ -125,23 +144,60 @@ const createBareServer = function () {
 };
 
 const enableOpsMetrics = async function (server) {
-  const oppsy = new Oppsy(server);
-
-  oppsy.on('ops', (data) => {
-    const knexPool = knex.client.pool;
-    server.log(['ops'], {
-      ...data,
-      knexPool: {
-        used: knexPool.numUsed(),
-        free: knexPool.numFree(),
-        pendingAcquires: knexPool.numPendingAcquires(),
-        pendingCreates: knexPool.numPendingCreates(),
-      },
+  function collectMemoryStats() {
+    const memUsage = process.memoryUsage();
+    metrics.gauge(`memory.rss`, memUsage.rss);
+    metrics.gauge('memory.heapTotal', memUsage.heapTotal);
+    metrics.gauge('memory.heapUsed', memUsage.heapUsed);
+  }
+
+  setInterval(collectMemoryStats, 5000);
+
+  const gaugeConnections = (pool) => () => {
+    metrics.gauge('db_connections_used', pool.numUsed());
+    metrics.gauge('db_connections_free', pool.numFree());
+    metrics.gauge('db_connections_pending_creation', pool.numPendingCreates());
+    metrics.gauge('db_connections_pending_detroy', pool['pendingDestroys'].length);
+  };
+
+  const client = knex.client;
+
+  client.pool.on('createSuccess', gaugeConnections(client.pool));
+  client.pool.on('acquireSuccess', gaugeConnections(client.pool));
+  client.pool.on('release', gaugeConnections(client.pool));
+  client.pool.on('destroySuccess', gaugeConnections(client.pool));
+
+  if (config.featureToggles.isDirectMetricsEnabled) {
+    server.events.on('response', (request) => {
+      const info = request.info;
+
+      const statusCode = request.raw.res.statusCode;
+      const responseTime = (info.completed !== undefined ? info.completed : info.responded) - info.received;
+
+      metrics.histogram(`responseTime.${request.route}.${statusCode}`, responseTime);
+      metrics.increment(`api-calls.${request.route}.${statusCode}`);
     });
-  });
+  }
 
-  oppsy.start(logging.opsEventIntervalInSeconds * 1000);
-  server.oppsy = oppsy;
+  if (!config.featureToggles.isOppsyDisabled) {
+    const oppsy = new Oppsy(server);
+
+    oppsy.on('ops', (data) => {
+      const knexPool = knex.client.pool;
+      server.log(['ops'], {
+        ...data,
+        knexPool: {
+          used: knexPool.numUsed(),
+          free: knexPool.numFree(),
+          pendingAcquires: knexPool.numPendingAcquires(),
+          pendingCreates: knexPool.numPendingCreates(),
+        },
+      });
+    });
+
+    oppsy.start(logging.opsEventIntervalInSeconds * 1000);
+    server.oppsy = oppsy;
+  }
 };
 
 const setupDeserialization = function (server) {

api/src/shared/config.js

@@ -204,10 +204,12 @@ const configuration = (function () {
       ),
       isAsyncQuestRewardingCalculationEnabled: toBoolean(process.env.FT_ENABLE_ASYNC_QUESTS_REWARDS_CALCULATION),
       isCertificationTokenScopeEnabled: toBoolean(process.env.FT_ENABLE_CERTIF_TOKEN_SCOPE),
+      isDirectMetricsEnabled: toBoolean(process.env.FT_ENABLE_DIRECT_METRICS),
       isNeedToAdjustCertificationAccessibilityEnabled: toBoolean(
         process.env.FT_ENABLE_NEED_TO_ADJUST_CERTIFICATION_ACCESSIBILITY,
       ),
       isNewAuthenticationDesignEnabled: toBoolean(process.env.FT_NEW_AUTHENTICATION_DESIGN_ENABLED),
+      isOppsyDisabled: toBoolean(process.env.FT_OPPSY_DISABLED),
       isPix1dEnabled: toBoolean(process.env.FT_PIX_1D_ENABLED),
       isPixCompanionEnabled: toBoolean(process.env.FT_PIX_COMPANION_ENABLED),
       isSelfAccountDeletionEnabled: toBoolean(process.env.FT_SELF_ACCOUNT_DELETION),
@@ -223,6 +225,8 @@ const configuration = (function () {
       enableRequestMonitoring: toBoolean(process.env.ENABLE_REQUEST_MONITORING),
     },
     infra: {
+      appName: process.env.APP,
+      containerName: process.env.CONTAINER,
       concurrencyForHeavyOperations: _getNumber(process.env.INFRA_CONCURRENCY_HEAVY_OPERATIONS, 2),
       chunkSizeForCampaignResultProcessing: _getNumber(process.env.INFRA_CHUNK_SIZE_CAMPAIGN_RESULT_PROCESSING, 10),
       chunkSizeForOrganizationLearnerDataProcessing: _getNumber(
@@ -290,6 +294,9 @@ const configuration = (function () {
         },
       },
     },
+    metrics: {
+      flushIntervalSeconds: _getNumber(process.env.DIRECT_METRICS_FLUSH_INTERVAL, 5),
+    },
     partner: {
       fetchTimeOut: ms(process.env.FETCH_TIMEOUT_MILLISECONDS || '20s'),
     },
@@ -412,7 +419,9 @@ const configuration = (function () {
     config.featureToggles.deprecatePoleEmploiPushNotification = false;
     config.featureToggles.isAlwaysOkValidateNextChallengeEndpointEnabled = false;
     config.featureToggles.isCertificationTokenScopeEnabled = false;
+    config.featureToggles.isDirectMetricsEnabled = false;
     config.featureToggles.isNeedToAdjustCertificationAccessibilityEnabled = false;
+    config.featureToggles.isOppsyDisabled = false;
     config.featureToggles.isPix1dEnabled = true;
     config.featureToggles.isPixCompanionEnabled = false;
     config.featureToggles.isSelfAccountDeletionEnabled = false;

api/src/shared/infrastructure/plugins/pino.js

@@ -71,15 +71,20 @@ const plugin = {
 
     server.events.on('response', (request) => {
       const info = request.info;
-      logger.info(
-        {
-          queryParams: request.query,
-          req: request,
-          res: request.raw.res,
-          responseTime: (info.completed !== undefined ? info.completed : info.responded) - info.received,
-        },
-        'request completed',
-      );
+
+      const shouldLog = !config.featureToggles.isDirectMetricsEnabled;
+
+      if (shouldLog || request.raw.res.statusCode != 200) {
+        logger.info(
+          {
+            queryParams: request.query,
+            req: request,
+            res: request.raw.res,
+            responseTime: (info.completed !== undefined ? info.completed : info.responded) - info.received,
+          },
+          'request completed',
+        );
+      }
     });
   },
 };