From 46000a00b26f59b68a81839d4acfea8c2116462e Mon Sep 17 00:00:00 2001 From: Steph0 Date: Wed, 27 Nov 2024 16:41:00 +0100 Subject: [PATCH 1/2] :sparkles: api: add CSV file support to rescore-certification script Co-authored-by: P-Jeremy < jemyplu@gmail.com> --- .../certification/rescore-certifications.js | 26 ++++++++++++++----- .../rescore-certifications_test.js | 8 +++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/api/scripts/certification/rescore-certifications.js b/api/scripts/certification/rescore-certifications.js index 5fed135f389..0db60379263 100644 --- a/api/scripts/certification/rescore-certifications.js +++ b/api/scripts/certification/rescore-certifications.js @@ -6,11 +6,19 @@ import { disconnect } from '../../db/knex-database-connection.js'; import { CertificationRescoringByScriptJob } from '../../src/certification/session-management/domain/models/CertificationRescoringByScriptJob.js'; import { certificationRescoringByScriptJobRepository } from '../../src/certification/session-management/infrastructure/repositories/jobs/certification-rescoring-by-script-job-repository.js'; import { logger } from '../../src/shared/infrastructure/utils/logger.js'; +import { parseCsv } from '../helpers/csvHelpers.js'; const modulePath = url.fileURLToPath(import.meta.url); const isLaunchedFromCommandLine = process.argv[1] === modulePath; -async function main(certificationCourseIds) { +/** + * Usage: node scripts/certification/rescore-certifications.js path/file.csv + * File has only one column of certification-courses.id (integer), no header + **/ +async function main(filePath) { + logger.info('Reading and parsing csv data file... '); + const certificationCourseIds = await extractCsvData(filePath); + logger.info(`Publishing ${certificationCourseIds.length} rescoring jobs`); const jobs = await _scheduleRescoringJobs(certificationCourseIds); @@ -24,6 +32,15 @@ async function main(certificationCourseIds) { return 0; } +async function extractCsvData(filePath) { + const dataRows = await parseCsv(filePath, { header: false, skipEmptyLines: true }); + return dataRows.reduce((certificationCourseIds, dataRow) => { + const certificationCenterId = parseInt(dataRow[0]); + certificationCourseIds.push(certificationCenterId); + return certificationCourseIds; + }, []); +} + const _scheduleRescoringJobs = async (certificationCourseIds) => { const promisefiedJobs = certificationCourseIds.map(async (certificationCourseId) => { try { @@ -40,11 +57,8 @@ const _scheduleRescoringJobs = async (certificationCourseIds) => { (async () => { if (isLaunchedFromCommandLine) { try { - const certificationCourseIds = process.argv[2] - .split(',') - .map((str) => parseInt(str, 10)) - .filter(Number.isInteger); - const exitCode = await main(certificationCourseIds); + const filePath = process.argv[2]; + const exitCode = await main(filePath); return exitCode; } catch (error) { logger.error(error); diff --git a/api/tests/integration/scripts/certification/rescore-certifications_test.js b/api/tests/integration/scripts/certification/rescore-certifications_test.js index 6597e2e11c7..bd38592adb8 100644 --- a/api/tests/integration/scripts/certification/rescore-certifications_test.js +++ b/api/tests/integration/scripts/certification/rescore-certifications_test.js @@ -1,13 +1,15 @@ import { main } from '../../../../scripts/certification/rescore-certifications.js'; -import { expect, knex } from '../../../test-helper.js'; +import { createTempFile, expect, knex } from '../../../test-helper.js'; describe('Integration | Scripts | Certification | rescore-certfication', function () { it('should save pg boss jobs for each certification course ids', async function () { // given - const certificationsCourseIdList = [1, 2]; + const file = 'certification-courses-ids-to-rescore.csv'; + const data = '1\n2\n'; + const csvFilePath = await createTempFile(file, data); // when - await main(certificationsCourseIdList); + await main(csvFilePath); // then const [job1, job2] = await knex('pgboss.job') From bcce378bac3579c1f834fb42dd019bcc84135e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Fran=C3=A7ois?= Date: Thu, 28 Nov 2024 16:56:33 +0100 Subject: [PATCH 2/2] :recycle: refactor: rewrite script to respect [pix/docs/adr/0056-standardisation-des-scripts-nodejs.md](https://github.com/1024pix/pix/blob/dev/docs/adr/0056-standardisation-des-scripts-nodejs.md) Co-Authored-By: GUL Co-Authored-By: GUL --- .../certification/rescore-certifications.js | 107 ++++++++---------- .../rescore-certifications_test.js | 28 ++++- 2 files changed, 70 insertions(+), 65 deletions(-) diff --git a/api/scripts/certification/rescore-certifications.js b/api/scripts/certification/rescore-certifications.js index 0db60379263..3a5bb8ad6a8 100644 --- a/api/scripts/certification/rescore-certifications.js +++ b/api/scripts/certification/rescore-certifications.js @@ -1,72 +1,61 @@ import 'dotenv/config'; -import * as url from 'node:url'; +import Joi from 'joi'; -import { disconnect } from '../../db/knex-database-connection.js'; import { CertificationRescoringByScriptJob } from '../../src/certification/session-management/domain/models/CertificationRescoringByScriptJob.js'; import { certificationRescoringByScriptJobRepository } from '../../src/certification/session-management/infrastructure/repositories/jobs/certification-rescoring-by-script-job-repository.js'; -import { logger } from '../../src/shared/infrastructure/utils/logger.js'; -import { parseCsv } from '../helpers/csvHelpers.js'; - -const modulePath = url.fileURLToPath(import.meta.url); -const isLaunchedFromCommandLine = process.argv[1] === modulePath; - -/** - * Usage: node scripts/certification/rescore-certifications.js path/file.csv - * File has only one column of certification-courses.id (integer), no header - **/ -async function main(filePath) { - logger.info('Reading and parsing csv data file... '); - const certificationCourseIds = await extractCsvData(filePath); - - logger.info(`Publishing ${certificationCourseIds.length} rescoring jobs`); - const jobs = await _scheduleRescoringJobs(certificationCourseIds); - - const errors = jobs.filter((result) => result.status === 'rejected'); - if (errors.length) { - errors.forEach((result) => logger.error(result.reason, 'Some jobs could not be published')); - return 1; +import { csvFileParser } from '../../src/shared/application/scripts/parsers.js'; +import { Script } from '../../src/shared/application/scripts/script.js'; +import { ScriptRunner } from '../../src/shared/application/scripts/script-runner.js'; + +const columnsSchemas = [{ name: 'certificationCourseId', schema: Joi.number() }]; + +export class RescoreCertificationScript extends Script { + constructor() { + super({ + description: 'Rescore all certification given by CSV file. This script will schedule job to rescore', + permanent: true, + options: { + file: { + type: 'string', + describe: + 'CSV File with only one column with certification-courses.id (integer) to process. Need `certificationCourseId`', + demandOption: true, + coerce: csvFileParser(columnsSchemas), + }, + }, + }); } - logger.info(`${jobs.length} jobs successfully published`); - return 0; -} + async handle({ options, logger }) { + const { file: certificationCourses } = options; + const certificationCourseIds = certificationCourses.map(({ certificationCourseId }) => certificationCourseId); -async function extractCsvData(filePath) { - const dataRows = await parseCsv(filePath, { header: false, skipEmptyLines: true }); - return dataRows.reduce((certificationCourseIds, dataRow) => { - const certificationCenterId = parseInt(dataRow[0]); - certificationCourseIds.push(certificationCenterId); - return certificationCourseIds; - }, []); -} + logger.info(`Publishing ${certificationCourseIds.length} rescoring jobs`); + const jobs = await this.#scheduleRescoringJobs(certificationCourseIds); -const _scheduleRescoringJobs = async (certificationCourseIds) => { - const promisefiedJobs = certificationCourseIds.map(async (certificationCourseId) => { - try { - await certificationRescoringByScriptJobRepository.performAsync( - new CertificationRescoringByScriptJob({ certificationCourseId }), - ); - } catch (error) { - throw new Error(`Error for certificationCourseId: [${certificationCourseId}]`, { cause: error }); + const errors = jobs.filter((result) => result.status === 'rejected'); + if (errors.length) { + errors.forEach((result) => logger.error(result.reason, 'Some jobs could not be published')); + return 1; } - }); - return Promise.allSettled(promisefiedJobs); -}; -(async () => { - if (isLaunchedFromCommandLine) { - try { - const filePath = process.argv[2]; - const exitCode = await main(filePath); - return exitCode; - } catch (error) { - logger.error(error); - process.exitCode = 1; - } finally { - await disconnect(); - } + logger.info(`${jobs.length} jobs successfully published`); + return 0; } -})(); -export { main }; + async #scheduleRescoringJobs(certificationCourseIds) { + const promisefiedJobs = certificationCourseIds.map(async (certificationCourseId) => { + try { + await certificationRescoringByScriptJobRepository.performAsync( + new CertificationRescoringByScriptJob({ certificationCourseId }), + ); + } catch (error) { + throw new Error(`Error for certificationCourseId: [${certificationCourseId}]`, { cause: error }); + } + }); + return Promise.allSettled(promisefiedJobs); + } +} + +await ScriptRunner.execute(import.meta.url, RescoreCertificationScript); diff --git a/api/tests/integration/scripts/certification/rescore-certifications_test.js b/api/tests/integration/scripts/certification/rescore-certifications_test.js index bd38592adb8..e49533d1bbb 100644 --- a/api/tests/integration/scripts/certification/rescore-certifications_test.js +++ b/api/tests/integration/scripts/certification/rescore-certifications_test.js @@ -1,15 +1,31 @@ -import { main } from '../../../../scripts/certification/rescore-certifications.js'; -import { createTempFile, expect, knex } from '../../../test-helper.js'; +import { RescoreCertificationScript } from '../../../../scripts/certification/rescore-certifications.js'; +import { createTempFile, expect, knex, sinon } from '../../../test-helper.js'; describe('Integration | Scripts | Certification | rescore-certfication', function () { - it('should save pg boss jobs for each certification course ids', async function () { - // given + it('should parse input file', async function () { + const script = new RescoreCertificationScript(); + const { options } = script.metaInfo; const file = 'certification-courses-ids-to-rescore.csv'; - const data = '1\n2\n'; + const data = 'certificationCourseId\n1\n2\n3\n'; const csvFilePath = await createTempFile(file, data); + const parsedData = await options.file.coerce(csvFilePath); + + expect(parsedData).to.deep.equals([ + { certificationCourseId: 1 }, + { certificationCourseId: 2 }, + { certificationCourseId: 3 }, + ]); + }); + + it('should save pg boss jobs for each certification course ids', async function () { + // given + const file = [{ certificationCourseId: 1 }, { certificationCourseId: 2 }]; + const logger = { info: sinon.spy(), error: sinon.spy() }; + const script = new RescoreCertificationScript(); + // when - await main(csvFilePath); + await script.handle({ logger, options: { file } }); // then const [job1, job2] = await knex('pgboss.job')