Skip to content

Commit

Permalink
[O2B-1393] Use certificate when handling lost runs and envs (#1793)
Browse files Browse the repository at this point in the history 
* [O2B-1393] Use certificate when handling lost runs and envs

* Fix linter

* Minor refactoring

* Remove grid mention
  • Loading branch information
@martinboulais
martinboulais authored Jan 24, 2025
1 parent 1a9f97b commit 2031879
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 78 deletions.
66 changes: 39 additions & 27 deletions lib/application.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,21 @@ const { webUiServer } = require('./server');
const { gRPCServer } = require('./server/index.js');
const { GRPCConfig, ServicesConfig } = require('./config');

const { monalisa: monalisaConfig, ccdb: ccdbConfig } = ServicesConfig;
const { userCertificate, monalisa: monalisaConfig, ccdb: ccdbConfig, enableHousekeeping } = ServicesConfig;
const { handleLostRunsAndEnvironments } = require('./server/services/housekeeping/handleLostRunsAndEnvironments.js');
const { isInTestMode } = require('./utilities/env-utils.js');
const { ScheduledProcessesManager } = require('./server/services/ScheduledProcessesManager.js');
const { MonAlisaSynchronizer } = require('./server/externalServicesSynchronization/monalisa/MonAlisaSynchronizer');
const { createMonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient');
const { LogManager } = require('@aliceo2/web-ui');
const { Kafka, logLevel } = require('kafkajs');
const { KafkaConfig } = require('./config/index.js');
const { AliEcsSynchronizer } = require('./server/kafka/AliEcsSynchronizer.js');
const { environmentService } = require('./server/services/environment/EnvironmentService.js');
const { runService } = require('./server/services/run/RunService.js');
const { CcdbSynchronizer } = require('./server/externalServicesSynchronization/ccdb/CcdbSynchronizer.js');
const { promises: fs } = require('fs');
const { MonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient.js');
const https = require('https');

/**
* Bookkeeping Application
Expand Down Expand Up @@ -78,15 +80,34 @@ class BookkeepingApplication {
await this.aliEcsSynchronizer.start();
}

if (monalisaConfig.enableSynchronization) {
const monAlisaSynchronizer = await this.createMonAlisaSynchronizer();
this.scheduledProcessesManager.schedule(
() => monAlisaSynchronizer.synchronize(),
{
wait: 10 * 1000,
every: monalisaConfig.synchronizationPeriod,
},
);
if (monalisaConfig.enableSynchronization || enableHousekeeping) {
const pfxCertificateBytes = await fs.readFile(userCertificate.path);
const certificatePassphrase = userCertificate.passphrase;

const httpAgent = pfxCertificateBytes && certificatePassphrase
? new https.Agent({ pfx: pfxCertificateBytes, passphrase: certificatePassphrase })
: undefined;

if (monalisaConfig.enableSynchronization) {
const monAlisaSynchronizer = await this.createMonAlisaSynchronizer(httpAgent);
this.scheduledProcessesManager.schedule(
() => monAlisaSynchronizer.synchronize(),
{
wait: 10 * 1000,
every: monalisaConfig.synchronizationPeriod,
},
);
}

if (enableHousekeeping) {
this.scheduledProcessesManager.schedule(
() => this.housekeeping(httpAgent),
{
wait: 30 * 1000,
every: 30 * 1000,
},
);
}
}

if (ccdbConfig.enableSynchronization) {
Expand All @@ -100,16 +121,6 @@ class BookkeepingApplication {
},
);
}

if (ServicesConfig.enableHousekeeping) {
this.scheduledProcessesManager.schedule(
() => this.housekeeping(),
{
wait: 30 * 1000,
every: 30 * 1000,
},
);
}
} catch (error) {
this._logger.errorMessage(`Error while starting: ${error}`);
return this.stop();
Expand All @@ -121,27 +132,28 @@ class BookkeepingApplication {
/**
* Instantiate MonAlisa synchronizer with global configuration
*
* @param {Agent} agent the HTTP agent to be used by synchronizer
* @return {Promise<MonAlisaSynchronizer>} resolves with MonAlisaSynchronizer instance
*/
async createMonAlisaSynchronizer() {
return new MonAlisaSynchronizer(await createMonAlisaClient({
async createMonAlisaSynchronizer(agent) {
return new MonAlisaSynchronizer(new MonAlisaClient({
dataPassesUrl: monalisaConfig.dataPassesUrl,
dataPassDetailsUrl: monalisaConfig.dataPassDetailsUrl,
simulationPassesUrl: monalisaConfig.simulationPassesUrl,
yearLowerLimit: monalisaConfig.dataPassesYearLowerLimit,
userCertificatePath: monalisaConfig.userCertificate.path,
certificatePassphrase: monalisaConfig.userCertificate.passphrase,
agent,
}));
}

/**
* Housekeeping method, it wraps @see handleLostRunsAndEnvironments and logs its results
*
* @param {Agent} httpAgent agent to be used by the HTTP client
* @return {Promise<void>} promise
*/
async housekeeping() {
async housekeeping(httpAgent) {
try {
const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments();
const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments(httpAgent);
const subMessages = [];
if (transitionedEnvironments.length > 0) {
subMessages.push(`environments (${transitionedEnvironments.join(', ')})`);
Expand Down
15 changes: 8 additions & 7 deletions lib/config/services.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
*/

const {
MONALISA_CERTIFICATE_PATH,
MONALISA_CERTIFICATE_PASSPHRASE,
USER_CERTIFICATE_PATH,
USER_CERTIFICATE_PASSPHRASE,
MONALISA_CERTIFICATE_PATH, // DEPRECATED
MONALISA_CERTIFICATE_PASSPHRASE, // DEPRECATED
DATA_PASSES_YEAR_LOWER_LIMIT,
MONALISA_DATA_PASSES_URL,
MONALISA_DATA_PASS_DETAILS_URL,
Expand All @@ -27,6 +29,10 @@ const {

exports.services = {
enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING?.toLowerCase() === 'true',
userCertificate: {
path: USER_CERTIFICATE_PATH ?? MONALISA_CERTIFICATE_PATH,
passphrase: USER_CERTIFICATE_PASSPHRASE ?? MONALISA_CERTIFICATE_PASSPHRASE,
},
aliEcsGui: {
url: process.env?.ALI_ECS_GUI_URL || null,
token: process.env?.ALI_ECS_GUI_TOKEN || null,
Expand All @@ -45,12 +51,7 @@ exports.services = {
aliFlpIndex: {
url: process.env?.ALI_FLP_INDEX_URL || null,
},

monalisa: {
userCertificate: {
path: MONALISA_CERTIFICATE_PATH,
passphrase: MONALISA_CERTIFICATE_PASSPHRASE,
},
dataPassesYearLowerLimit: Number(DATA_PASSES_YEAR_LOWER_LIMIT) || 2022,

dataPassesUrl: MONALISA_DATA_PASSES_URL,
Expand Down
45 changes: 6 additions & 39 deletions lib/server/externalServicesSynchronization/monalisa/MonAlisaClient.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
* or submit itself to any jurisdiction.
*/

const https = require('https');
const { extractLhcPeriod } = require('../../utilities/extractLhcPeriod');
const fs = require('fs').promises;

const VALID_DATA_PASS_NAME_REGEX = /^LHC\d\d[a-zA-Z]+_([a-z]pass|skimming|skimmed)[^\s]*$/;

Expand Down Expand Up @@ -62,20 +60,17 @@ class MonAlisaClient {
* @param {object} configuration configuration of MonAlisa client
* @param {URL|string} [configuration.dataPassesUrl] url to fetch data passes
* @param {URL|string} [configuration.dataPassDetailsUrl] url to fetch data pass version details
* @param {string|string[]|Buffer|Buffer[]|Object[]} [configuration.pfxCertificateBytes] PFX or PKCS12 encoded private key
* and certificate chain @see tls.createSecureContext([options])
* @param {string} [configuration.certificatePassphrase] passphrase to the certificate
* @param {URL|string} [configuration.simulationPassesUrl] url to fetch simulation passes
* @param {number} [configuration.yearLowerLimit] indicates how old data are accepted,
* year of lhc period that given data pass belongs must be greater or equal
* @param {URL|string} [configuration.simulationPassesUrl] url to fetch simulation passes
* @param {Agent} agent HTTPs agent to use when sending requests
*/
constructor({
dataPassesUrl,
dataPassDetailsUrl,
pfxCertificateBytes,
certificatePassphrase,
yearLowerLimit,
simulationPassesUrl,
yearLowerLimit,
agent,
} = {}) {
this.dataPassesUrl = dataPassesUrl ? new URL(dataPassesUrl) : null;
this.dataPassDetailsUrl = dataPassDetailsUrl ? new URL(dataPassDetailsUrl) : null;
Expand All @@ -88,8 +83,7 @@ class MonAlisaClient {
Accept: 'application/json;charset=utf-8',
Connection: 'keep-alive',
},
agent: pfxCertificateBytes && certificatePassphrase
? new https.Agent({ pfx: pfxCertificateBytes, passphrase: certificatePassphrase }) : undefined,
agent,
};
}

Expand All @@ -106,7 +100,7 @@ class MonAlisaClient {

/**
* Fetch data passes versions from MonAlisa
* @return {string} payload raw data acquired from MonAlisa: csv with data passes properties
* @return {Promise<string>} payload raw data acquired from MonAlisa: csv with data passes properties
* @private
*/
async _fetchDataPassesVersions() {
Expand Down Expand Up @@ -283,30 +277,3 @@ class MonAlisaClient {
}

exports.MonAlisaClient = MonAlisaClient;

/**
* Create a new instance of MonAlisaClient
*
* @param {object} configuration configuration of MonAlisa client
* @param {URL|string} [configuration.dataPassesUrl] url to fetch data passes
* @param {URL|string} [configuration.dataPassDetailsUrl] url to fetch data pass version details
* @param {number} [configuration.yearLowerLimit] indicates how old data are accepted,
* @param {string} [configuration.userCertificatePath] path to PKCS12 certificate
* @param {string} [configuration.certificatePassphrase] passphrase to the certificate
* @return {Promise<MonAlisaClient>} resolved with instance configured with environment variables
*/
exports.createMonAlisaClient = async ({
dataPassesUrl,
dataPassDetailsUrl,
simulationPassesUrl,
yearLowerLimit,
userCertificatePath,
certificatePassphrase,
}) => new MonAlisaClient({
dataPassesUrl,
dataPassDetailsUrl,
simulationPassesUrl,
yearLowerLimit,
pfxCertificateBytes: await fs.readFile(userCertificatePath),
certificatePassphrase,
});
15 changes: 10 additions & 5 deletions lib/server/services/housekeeping/handleLostRunsAndEnvironments.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,21 @@ const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;
* mark them as gone to error. For all runs that do not have timeO2Stop or timeTrgStop, check through AliECS GUI if they are still running, and
* if not, mark them as stopped NOW
*
* @param {Agent} [httpAgent] agent to be used when sending HTTP requests
* @return {Promise<{transitionedEnvironments: number[], endedRuns: []}>} resolve with the list of environment ids and run numbers that were lost
* @deprecated
*/
exports.handleLostRunsAndEnvironments = async () => {
exports.handleLostRunsAndEnvironments = async (httpAgent) => {
// TODO remove with node 18
const { default: fetch } = await import('node-fetch');
const existingEnvironmentsResponse = await fetch(buildUrl(
`${ServicesConfig.aliEcsGui.url}/api/core/environments`,
{ token: ServicesConfig.aliEcsGui.token },
));

const existingEnvironmentsResponse = await fetch(
buildUrl(
`${ServicesConfig.aliEcsGui.url}/api/core/environments`,
{ token: ServicesConfig.aliEcsGui.token },
),
{ agent: httpAgent },
);

if (existingEnvironmentsResponse.ok) {
const { environments } = await existingEnvironmentsResponse.json();
Expand Down

0 comments on commit 2031879

Please sign in to comment.