diff --git a/api/db/migrations/20240924091540_add-reconciliation-date-to-former-candidates.js b/api/db/migrations/20240924091540_add-reconciliation-date-to-former-candidates.js index dbd433a90ca..93c734fdf3a 100644 --- a/api/db/migrations/20240924091540_add-reconciliation-date-to-former-candidates.js +++ b/api/db/migrations/20240924091540_add-reconciliation-date-to-former-candidates.js @@ -2,6 +2,12 @@ import { logger } from '../../src/shared/infrastructure/utils/logger.js'; const TABLE_NAME = 'certification-candidates'; +/** + ************************************************** + * WARNING: this migration has a flaw, and did not + * perform fully + * ************************************************ + */ const up = async function (knex) { let numberOfBatchProcessed = 0; const CHUNK_SIZE = 250000; diff --git a/api/scripts/certification/finishing-reconciled-at-migration.js b/api/scripts/certification/finishing-reconciled-at-migration.js new file mode 100644 index 00000000000..a106ff01870 --- /dev/null +++ b/api/scripts/certification/finishing-reconciled-at-migration.js @@ -0,0 +1,39 @@ +import 'dotenv/config'; + +import * as url from 'node:url'; + +import { disconnect as disconnectFromDb } from '../../db/knex-database-connection.js'; +import { usecases } from '../../src/certification/configuration/domain/usecases/index.js'; +import { logger } from '../../src/shared/infrastructure/utils/logger.js'; + +/** + * This migration re-perform the action from api/db/migrations/20240920163000_modify-to-reconciledAt-on-certification-candidates-table .js + * + * Usage: CHUNK_SIZE=100000 node scripts/certification/finishing-reconciled-at-migration.js + **/ + +const modulePath = url.fileURLToPath(import.meta.url); +const isLaunchedFromCommandLine = process.argv[1] === modulePath; + +const main = async ({ chunkSize }) => { + return usecases.catchingUpCandidateReconciliation({ chunkSize }); +}; + +(async () => { + if (isLaunchedFromCommandLine) { + let exitCode = 0; + try { + const chunkSize = parseInt(process.env.CHUNK_SIZE, 10); + await main({ chunkSize: isNaN(chunkSize) ? undefined : chunkSize }); + } catch (error) { + logger.error(error); + exitCode = 1; + } finally { + await disconnectFromDb(); + // eslint-disable-next-line n/no-process-exit + process.exit(exitCode); + } + } +})(); + +export { main }; diff --git a/api/src/certification/configuration/domain/models/Candidate.js b/api/src/certification/configuration/domain/models/Candidate.js new file mode 100644 index 00000000000..d289cf0b2f6 --- /dev/null +++ b/api/src/certification/configuration/domain/models/Candidate.js @@ -0,0 +1,29 @@ +import Joi from 'joi'; + +import { EntityValidationError } from '../../../../shared/domain/errors.js'; + +export class Candidate { + static #schema = Joi.object({ + id: Joi.number().integer().required(), + reconciledAt: Joi.date().required(), + }); + + /** + * @param {Object} props + * @param {number} props.id + * @param {Date} props.reconciledAt + */ + constructor({ id, reconciledAt }) { + this.id = id; + this.reconciledAt = reconciledAt; + + this.#validate(); + } + + #validate() { + const { error } = Candidate.#schema.validate(this, { allowUnknown: false }); + if (error) { + throw EntityValidationError.fromJoiErrors(error.details); + } + } +} diff --git a/api/src/certification/configuration/domain/usecases/catching-up-candidate-reconciliation.js b/api/src/certification/configuration/domain/usecases/catching-up-candidate-reconciliation.js new file mode 100644 index 00000000000..eab30d0f561 --- /dev/null +++ b/api/src/certification/configuration/domain/usecases/catching-up-candidate-reconciliation.js @@ -0,0 +1,44 @@ +/** + * @typedef {import ('./index.js').CandidateRepository} CandidateRepository + * @typedef {import ('../models/Candidate.js').Candidate} Candidate + */ + +import { DomainTransaction } from '../../../../shared/domain/DomainTransaction.js'; +import { logger } from '../../../../shared/infrastructure/utils/logger.js'; + +/** + * @param {Object} params + * @param {number} params.[chunkSize] - default 100000 + * @param {CandidateRepository} params.candidateRepository + * @returns {Promise} + */ +export const catchingUpCandidateReconciliation = async ({ chunkSize = 100000, candidateRepository }) => { + logger.info(`Starting certification-candidates.reconciledAt updates by chunk of ${chunkSize}`); + let migratedLines = 0; + let hasNext = true; + do { + const candidates = await candidateRepository.findCandidateWithoutReconciledAt({ limit: chunkSize }); + logger.info(`Found ${candidates.length} certification-candidates without reconciledAt`); + + migratedLines += await performUpdates({ candidates, candidateRepository }); + logger.info(`Update committed for ${migratedLines} candidates`); + hasNext = !!candidates.length; + } while (hasNext); + + logger.info(`Total of ${migratedLines} certification-candidates.reconciledAt updated`); +}; + +/** + * @param {Object} params + * @param {Array} params.candidates + * @param {CandidateRepository} params.candidateRepository + */ +const performUpdates = async ({ candidates = [], candidateRepository }) => { + return DomainTransaction.execute(async () => { + let migratedLines = 0; + for (const candidate of candidates) { + migratedLines += await candidateRepository.update({ candidate }); + } + return migratedLines; + }); +}; diff --git a/api/src/certification/configuration/domain/usecases/index.js b/api/src/certification/configuration/domain/usecases/index.js index 768b8e0b479..162b41e41a4 100644 --- a/api/src/certification/configuration/domain/usecases/index.js +++ b/api/src/certification/configuration/domain/usecases/index.js @@ -5,6 +5,7 @@ import { injectDependencies } from '../../../../shared/infrastructure/utils/depe import { importNamedExportsFromDirectory } from '../../../../shared/infrastructure/utils/import-named-exports-from-directory.js'; import * as complementaryCertificationRepository from '../../../complementary-certification/infrastructure/repositories/complementary-certification-repository.js'; import * as attachableTargetProfileRepository from '../../infrastructure/repositories/attachable-target-profiles-repository.js'; +import * as candidateRepository from '../../infrastructure/repositories/candidate-repository.js'; import * as centerPilotFeaturesRepository from '../../infrastructure/repositories/center-pilot-features-repository.js'; import * as centerRepository from '../../infrastructure/repositories/center-repository.js'; import * as habilitationRepository from '../../infrastructure/repositories/habilitation-repository.js'; @@ -25,6 +26,7 @@ import { convertCenterToV3JobRepository } from '../../infrastructure/repositorie * @typedef {convertCenterToV3JobRepository} ConvertCenterToV3JobRepository * @typedef {sessionsRepository} SessionsRepository * @typedef {habilitationRepository} HabilitationRepository + * @typedef {candidateRepository} CandidateRepository **/ const dependencies = { attachableTargetProfileRepository, @@ -34,6 +36,7 @@ const dependencies = { convertCenterToV3JobRepository, sessionsRepository: configurationRepositories.sessionsRepository, habilitationRepository, + candidateRepository, }; const path = dirname(fileURLToPath(import.meta.url)); diff --git a/api/src/certification/configuration/infrastructure/repositories/candidate-repository.js b/api/src/certification/configuration/infrastructure/repositories/candidate-repository.js new file mode 100644 index 00000000000..779a5950290 --- /dev/null +++ b/api/src/certification/configuration/infrastructure/repositories/candidate-repository.js @@ -0,0 +1,48 @@ +import { DomainTransaction } from '../../../../shared/domain/DomainTransaction.js'; +import { Candidate } from '../../domain/models/Candidate.js'; + +/** + * This function find candidates with a certification-course but no reconciledAt + * + * @param {Object} params + * @param {number} params.[limit] - number of candidates to limit to + * @returns {Array} - Candidates returned have a reconciledAt built from certification-course + */ +export const findCandidateWithoutReconciledAt = async function ({ limit } = {}) { + const knexConn = DomainTransaction.getConnection(); + const data = await knexConn('certification-candidates') + .select('certification-candidates.id', 'certification-courses.createdAt') + .where((queryBuilder) => { + queryBuilder.whereNotNull('certification-candidates.userId'); + }) + .andWhere((queryBuilder) => { + queryBuilder.whereNull('certification-candidates.reconciledAt'); + }) + .innerJoin('certification-courses', function () { + this.on('certification-courses.userId', 'certification-candidates.userId').andOn( + 'certification-courses.sessionId', + 'certification-candidates.sessionId', + ); + }) + .limit(limit); + + return data.map((data) => _toDomain({ id: data.id, reconciledAt: data.createdAt })); +}; + +/** + * @param {Object} params + * @param {Candidate} params.candidate + * @returns {number} - number of rows affected + */ +export const update = async function ({ candidate }) { + const knexConn = DomainTransaction.getConnection(); + const results = await knexConn('certification-candidates') + .update({ reconciledAt: candidate.reconciledAt }) + .where({ id: candidate.id }); + + return results || 0; +}; + +const _toDomain = ({ id, reconciledAt }) => { + return new Candidate({ id, reconciledAt }); +}; diff --git a/api/tests/certification/configuration/integration/infrastructure/repositories/candidate-repository_test.js b/api/tests/certification/configuration/integration/infrastructure/repositories/candidate-repository_test.js new file mode 100644 index 00000000000..7bfe8d9c57a --- /dev/null +++ b/api/tests/certification/configuration/integration/infrastructure/repositories/candidate-repository_test.js @@ -0,0 +1,94 @@ +import { Candidate } from '../../../../../../src/certification/configuration/domain/models/Candidate.js'; +import * as candidateRepository from '../../../../../../src/certification/configuration/infrastructure/repositories/candidate-repository.js'; +import { databaseBuilder, expect, knex } from '../../../../../test-helper.js'; + +describe('Certification | Configuration | Integration | Repository | candidate-repository', function () { + describe('findCandidateWithoutReconciledAt', function () { + it('should find candidate with a course and no reconciledAt', async function () { + // given + const courseDate = new Date(); + const sessionId = databaseBuilder.factory.buildSession({}).id; + const userId = databaseBuilder.factory.buildUser().id; + databaseBuilder.factory.buildCertificationCourse({ userId, sessionId, createdAt: courseDate }); + const candidateId = databaseBuilder.factory.buildCertificationCandidate({ + userId, + sessionId, + reconciledAt: null, + }).id; + await databaseBuilder.commit(); + // this is necessary to introduce the problem we want to find in the current database + await knex('certification-candidates').update({ reconciledAt: null }).where({ userId }); + + // when + const results = await candidateRepository.findCandidateWithoutReconciledAt(); + + // then + expect(results).to.deep.equal([new Candidate({ id: candidateId, reconciledAt: courseDate })]); + }); + + it('should find candidate with no course and no reconciledAt', async function () { + // given + const userId = databaseBuilder.factory.buildUser().id; + const sessionId = databaseBuilder.factory.buildSession({ userId }).id; + databaseBuilder.factory.buildCertificationCandidate({ + userId, + sessionId, + reconciledAt: null, + }); + await databaseBuilder.commit(); + // this is necessary to introduce the problem we want to find in the current database + await knex('certification-candidates').update({ reconciledAt: null }).where({ userId }); + + // when + const results = await candidateRepository.findCandidateWithoutReconciledAt(); + + // then + expect(results).to.be.empty; + }); + + it('should not find candidate with a course and already a reconciledAt', async function () { + // given + const userId = databaseBuilder.factory.buildUser().id; + const sessionId = databaseBuilder.factory.buildSession({ userId }).id; + databaseBuilder.factory.buildCertificationCourse({ userId, sessionId, createdAt: new Date() }); + databaseBuilder.factory.buildCertificationCandidate({ + userId, + sessionId, + reconciledAt: new Date(), + }); + await databaseBuilder.commit(); + + // when + const results = await candidateRepository.findCandidateWithoutReconciledAt(); + + // then + expect(results).to.be.empty; + }); + }); + + describe('update', function () { + it('should update reconciledAt when modified', async function () { + // given + const oldReconciledDate = new Date('2024-01-01'); + const newReconciledDate = new Date('2024-10-29'); + const userId = databaseBuilder.factory.buildUser().id; + const sessionId = databaseBuilder.factory.buildSession({ userId }).id; + const candidateId = databaseBuilder.factory.buildCertificationCandidate({ + userId, + sessionId, + reconciledAt: oldReconciledDate, + }).id; + await databaseBuilder.commit(); + + // when + const rowChanged = await candidateRepository.update({ + candidate: new Candidate({ id: candidateId, reconciledAt: newReconciledDate }), + }); + + // then + expect(rowChanged).to.equal(1); + const reconciledAt = await knex('certification-candidates').pluck('reconciledAt').where({ id: candidateId }); + expect(reconciledAt[0]).to.deep.equal(newReconciledDate); + }); + }); +}); diff --git a/api/tests/integration/scripts/certification/finishing-reconciled-at-migration_test.js b/api/tests/integration/scripts/certification/finishing-reconciled-at-migration_test.js new file mode 100644 index 00000000000..bc018d4edbd --- /dev/null +++ b/api/tests/integration/scripts/certification/finishing-reconciled-at-migration_test.js @@ -0,0 +1,54 @@ +import { main } from '../../../../scripts/certification/finishing-reconciled-at-migration.js'; +import { databaseBuilder, expect, knex } from '../../../test-helper.js'; + +describe('Integration | Scripts | Certification | finishing-reconciled-at-migration', function () { + it('should fix the missing reconciledAt', async function () { + // given + const courseDate = new Date('2024-10-29'); + const sessionId = databaseBuilder.factory.buildSession({}).id; + + const userIdOne = databaseBuilder.factory.buildUser().id; + databaseBuilder.factory.buildCertificationCourse({ userId: userIdOne, sessionId, createdAt: courseDate }); + const candidateIdOne = databaseBuilder.factory.buildCertificationCandidate({ + userId: userIdOne, + sessionId, + }).id; + + const userIdTwo = databaseBuilder.factory.buildUser().id; + databaseBuilder.factory.buildCertificationCourse({ userId: userIdTwo, sessionId, createdAt: courseDate }); + const candidateIdTwo = databaseBuilder.factory.buildCertificationCandidate({ + userId: userIdTwo, + sessionId, + }).id; + + const userIdThree = databaseBuilder.factory.buildUser().id; + const candidateIdWithNoCourse = databaseBuilder.factory.buildCertificationCandidate({ + userId: userIdThree, + sessionId, + }).id; + await databaseBuilder.commit(); + + // Creating the issue we wanna fix in database + await knex('certification-candidates') + .update({ reconciledAt: null }) + .whereIn('id', [candidateIdOne, candidateIdTwo, candidateIdWithNoCourse]); + + // when + await main({ chunkSize: 1 }); + + // then + const reconciledAts = await knex('certification-candidates') + .pluck('reconciledAt') + .whereIn('id', [candidateIdOne, candidateIdTwo]); + + expect(reconciledAts).to.have.lengthOf(2); + for (const reconciledAt of reconciledAts) { + expect(reconciledAt).to.deep.equal(courseDate); + } + + const reconciledAtForNoCourse = await knex('certification-candidates') + .pluck('reconciledAt') + .where('id', candidateIdWithNoCourse); + expect(reconciledAtForNoCourse[0]).to.be.null; + }); +});