[TECH] add direct metrics pushed to datadog

api/index.js

@@ -4,6 +4,8 @@ import { validateEnvironmentVariables } from './src/shared/infrastructure/valida
 
 validateEnvironmentVariables();
 
+import metrics from 'datadog-metrics';
+
 import { disconnect, prepareDatabaseConnection } from './db/knex-database-connection.js';
 import { createServer } from './server.js';
 import { config } from './src/shared/config.js';
@@ -35,6 +37,11 @@ async function _exitOnSignal(signal) {
   logger.info(`Received signal: ${signal}.`);
   logger.info('Stopping HAPI server...');
   await server.stop({ timeout: 30000 });
+  if (server.pixCustomIntervals) {
+    logger.info('Closing metrics interval and flush metrics');
+    server.pixCustomIntervals.forEach(clearInterval);
+    metrics.flush();
+  }
   if (server.oppsy) {
     logger.info('Stopping HAPI Oppsy server...');
     await server.oppsy.stop();

api/package.json

@@ -39,6 +39,7 @@
     "cron-parser": "^4.9.0",
     "dataloader": "^2.2.2",
     "dayjs": "^1.11.5",
+    "datadog-metrics": "^0.12.0",
     "debug": "^4.3.4",
     "dotenv": "^16.0.1",
     "fast-levenshtein": "^3.0.0",

api/sample.env

@@ -49,6 +49,14 @@ REDIS_URL=redis://localhost:6379
 # default: none
 # CACHE_RELOAD_CONTAINER_SIZE=M
 
+# =====
+# DATADOG
+# =====
+
+# Datadog access context
+#
+DATADOG_API_KEY=nokey
+
 # =========
 # DATABASES
 # =========

api/server.js

@@ -1,5 +1,6 @@
 import Oppsy from '@1024pix/oppsy';
 import Hapi from '@hapi/hapi';
+import metrics from 'datadog-metrics';
 import { parse } from 'neoqs';
 
 import { setupErrorHandling } from './config/server-setup-error-handling.js';
@@ -36,6 +37,7 @@ import { config } from './src/shared/config.js';
 import { monitoringTools } from './src/shared/infrastructure/monitoring-tools.js';
 import { plugins } from './src/shared/infrastructure/plugins/index.js';
 import { deserializer } from './src/shared/infrastructure/serializers/jsonapi/deserializer.js';
+import { logger } from './src/shared/infrastructure/utils/logger.js';
 // bounded context migration
 import { sharedRoutes } from './src/shared/routes.js';
 import { swaggers } from './src/shared/swaggers.js';
@@ -70,7 +72,25 @@ const { logOpsMetrics, port, logging } = config;
 const createServer = async () => {
   const server = createBareServer();
 
-  if (logOpsMetrics) await enableOpsMetrics(server);
+  // initialisation of Datadog link for metrics publication
+  if (config.environment !== 'development' || config.featureToggles.isDirectMetricsEnabled) {
+    logger.info('Metric initialisation : linked to Datadog');
+    metrics.init({
+      host: config.infra.containerName,
+      prefix: '',
+      defaultTags: [`service:${config.infra.appName}`],
+    });
+  } else {
+    logger.info('Metric initialisation : no reporter => no metrics sent');
+    metrics.init({ reporter: metrics.NullReporter() });
+  }
+
+  if (logOpsMetrics) {
+    // OPS metrics via direct metrics
+    if (config.featureToggles.isDirectMetricsEnabled) await enableOpsMetrics(server);
+    // OPS metrics via Oppsy
+    if (!config.featureToggles.isOppsyDisabled) await enableLegacyOpsMetrics(server);
+  }
 
   setupErrorHandling(server);
 
@@ -126,6 +146,48 @@ const createBareServer = function () {
 };
 
 const enableOpsMetrics = async function (server) {
+  function collectMemoryStats() {
+    const memUsage = process.memoryUsage();
+    metrics.gauge(`captain.api.memory.rss`, memUsage.rss);
+    metrics.gauge('captain.api.memory.heapTotal', memUsage.heapTotal);
+    metrics.gauge('captain.api.memory.heapUsed', memUsage.heapUsed);
+    metrics.gauge('captain.api.conteneur', 1);
+  }
+
+  const metricsInterval = setInterval(collectMemoryStats, 1000);
+  server.pixCustomIntervals = [metricsInterval];
+
+  const gaugeConnections = (pool) => () => {
+    logger.info('publie les metrics de knex');
+    metrics.gauge('captain.api.knex.db_connections_used', pool.numUsed());
+    metrics.gauge('captain.api.knex.db_connections_free', pool.numFree());
+    metrics.gauge('captain.api.knex.db_connections_pending_creation', pool.numPendingCreates());
+    metrics.gauge('captain.api.knex.db_connections_pending_detroy', pool['pendingDestroys'].length);
+  };
+
+  const client = knex.client;
+  gaugeConnections(client.pool)();
+
+  client.pool.on('createSuccess', gaugeConnections(client.pool));
+  client.pool.on('acquireSuccess', gaugeConnections(client.pool));
+  client.pool.on('release', gaugeConnections(client.pool));
+  client.pool.on('destroySuccess', gaugeConnections(client.pool));
+
+  server.events.on('response', (request) => {
+    const info = request.info;
+
+    const statusCode = request.raw.res.statusCode;
+    const responseTime = (info.completed !== undefined ? info.completed : info.responded) - info.received;
+
+    metrics.histogram('captain.api.duration', responseTime, [
+      `method:${request.route.method}`,
+      `route:${request.route.path}`,
+      `statusCode:${statusCode}`,
+    ]);
+  });
+};
+
+const enableLegacyOpsMetrics = async function (server) {
   const oppsy = new Oppsy(server);
 
   oppsy.on('ops', (data) => {

api/src/shared/config.js

@@ -251,9 +251,11 @@ const configuration = (function () {
         process.env.FT_ALWAYS_OK_VALIDATE_NEXT_CHALLENGE_ENDPOINT,
       ),
       isAsyncQuestRewardingCalculationEnabled: toBoolean(process.env.FT_ENABLE_ASYNC_QUESTS_REWARDS_CALCULATION),
+      isDirectMetricsEnabled: toBoolean(process.env.FT_ENABLE_DIRECT_METRICS),
       isNeedToAdjustCertificationAccessibilityEnabled: toBoolean(
         process.env.FT_ENABLE_NEED_TO_ADJUST_CERTIFICATION_ACCESSIBILITY,
       ),
+      isOppsyDisabled: toBoolean(process.env.FT_OPPSY_DISABLED),
       isPix1dEnabled: toBoolean(process.env.FT_PIX_1D_ENABLED),
       isPixCompanionEnabled: toBoolean(process.env.FT_PIX_COMPANION_ENABLED),
       isSelfAccountDeletionEnabled: toBoolean(process.env.FT_SELF_ACCOUNT_DELETION),
@@ -270,6 +272,8 @@ const configuration = (function () {
       enableRequestMonitoring: toBoolean(process.env.ENABLE_REQUEST_MONITORING),
     },
     infra: {
+      appName: process.env.APP,
+      containerName: process.env.CONTAINER,
       concurrencyForHeavyOperations: _getNumber(process.env.INFRA_CONCURRENCY_HEAVY_OPERATIONS, 2),
       chunkSizeForCampaignResultProcessing: _getNumber(process.env.INFRA_CHUNK_SIZE_CAMPAIGN_RESULT_PROCESSING, 10),
       chunkSizeForOrganizationLearnerDataProcessing: _getNumber(
@@ -344,6 +348,9 @@ const configuration = (function () {
         },
       },
     },
+    metrics: {
+      flushIntervalSeconds: _getNumber(process.env.DIRECT_METRICS_FLUSH_INTERVAL, 5),
+    },
     partner: {
       fetchTimeOut: ms(process.env.FETCH_TIMEOUT_MILLISECONDS || '20s'),
     },
@@ -478,7 +485,9 @@ const configuration = (function () {
 
     config.featureToggles.deprecatePoleEmploiPushNotification = false;
     config.featureToggles.isAlwaysOkValidateNextChallengeEndpointEnabled = false;
+    config.featureToggles.isDirectMetricsEnabled = false;
     config.featureToggles.isNeedToAdjustCertificationAccessibilityEnabled = false;
+    config.featureToggles.isOppsyDisabled = false;
     config.featureToggles.isPix1dEnabled = true;
     config.featureToggles.isPixCompanionEnabled = false;
     config.featureToggles.isSelfAccountDeletionEnabled = false;

api/src/shared/infrastructure/plugins/pino.js

@@ -74,15 +74,20 @@ const plugin = {
 
     server.events.on('response', (request) => {
       const info = request.info;
-      logger.info(
-        {
-          queryParams: request.query,
-          req: request,
-          res: request.raw.res,
-          responseTime: (info.completed !== undefined ? info.completed : info.responded) - info.received,
-        },
-        'request completed',
-      );
+
+      const shouldLog = !config.featureToggles.isDirectMetricsEnabled;
+
+      if (shouldLog || request.raw.res.statusCode != 200) {
+        logger.info(
+          {
+            queryParams: request.query,
+            req: request,
+            res: request.raw.res,
+            responseTime: (info.completed !== undefined ? info.completed : info.responded) - info.received,
+          },
+          'request completed',
+        );
+      }
     });
   },
 };

api/tests/shared/acceptance/application/feature-toggles/feature-toggle-controller_test.js

@@ -23,7 +23,9 @@ describe('Acceptance | Shared | Application | Controller | feature-toggle', func
             'deprecate-pole-emploi-push-notification': false,
             'is-always-ok-validate-next-challenge-endpoint-enabled': false,
             'is-async-quest-rewarding-calculation-enabled': false,
+            'is-direct-metrics-enabled': false,
             'is-need-to-adjust-certification-accessibility-enabled': false,
+            'is-oppsy-disabled': false,
             'is-pix1d-enabled': true,
             'is-pix-companion-enabled': false,
             'is-quest-enabled': false,