[FEATURE] Rendre asynchrone l'import à format sur PixOrga (Pix-15437)

api/src/prescription/learner-management/application/jobs/import-common-organization-learners-job-controller.js

@@ -0,0 +1,34 @@
+import { JobController } from '../../../../shared/application/jobs/job-controller.js';
+import { config } from '../../../../shared/config.js';
+import { DomainError } from '../../../../shared/domain/errors.js';
+import { logger as l } from '../../../../shared/infrastructure/utils/logger.js';
+import { ImportCommonOrganizationLearnersJob } from '../../domain/models/ImportCommonOrganizationLearnersJob.js';
+import { usecases } from '../../domain/usecases/index.js';
+
+class ImportCommonOrganizationLearnersJobController extends JobController {
+  #logger;
+
+  constructor({ logger = l } = {}) {
+    super(ImportCommonOrganizationLearnersJob.name);
+    this.#logger = logger;
+  }
+
+  get isJobEnabled() {
+    return config.pgBoss.importFileJobEnabled;
+  }
+
+  async handle({ data }) {
+    const { organizationImportId } = data;
+
+    try {
+      return await usecases.saveOrganizationLearnersFile({ organizationImportId });
+    } catch (err) {
+      if (!(err instanceof DomainError)) {
+        throw err;
+      }
+      this.#logger.error(err);
+    }
+  }
+}
+
+export { ImportCommonOrganizationLearnersJobController };

api/src/prescription/learner-management/application/jobs/validate-common-organization-learners-import-file-job-controller.js

@@ -0,0 +1,33 @@
+import { JobController } from '../../../../shared/application/jobs/job-controller.js';
+import { config } from '../../../../shared/config.js';
+import { DomainError } from '../../../../shared/domain/errors.js';
+import { logger as l } from '../../../../shared/infrastructure/utils/logger.js';
+import { ValidateCommonOrganizationImportFileJob } from '../../domain/models/ValidateCommonOrganizationImportFileJob.js';
+import { usecases } from '../../domain/usecases/index.js';
+
+class ValidateCommonOrganizationLearnersImportFileJobController extends JobController {
+  #logger;
+
+  constructor({ logger = l } = {}) {
+    super(ValidateCommonOrganizationImportFileJob.name);
+    this.#logger = logger;
+  }
+
+  get isJobEnabled() {
+    return config.pgBoss.validationFileJobEnabled;
+  }
+
+  async handle({ data }) {
+    const { organizationImportId } = data;
+    try {
+      await usecases.validateOrganizationLearnersFile({ organizationImportId });
+    } catch (err) {
+      if (!(err instanceof DomainError)) {
+        throw err;
+      }
+      this.#logger.warn(err);
+    }
+  }
+}
+
+export { ValidateCommonOrganizationLearnersImportFileJobController };

api/src/prescription/learner-management/application/organization-learners-controller.js

@@ -1,5 +1,4 @@
 import { DomainTransaction } from '../../../shared/domain/DomainTransaction.js';
-import { ApplicationTransaction } from '../../shared/infrastructure/ApplicationTransaction.js';
 import { usecases } from '../domain/usecases/index.js';
 import * as scoOrganizationLearnerSerializer from '../infrastructure/serializers/jsonapi/sco-organization-learner-serializer.js';
 
@@ -22,15 +21,7 @@ const importOrganizationLearnerFromFeature = async function (request, h) {
   const organizationId = request.params.organizationId;
   const userId = request.auth.credentials.userId;
 
-  await ApplicationTransaction.execute(async () => {
-    await usecases.sendOrganizationLearnersFile({ payload: request.payload, organizationId, userId });
-  });
-  await ApplicationTransaction.execute(async () => {
-    await usecases.validateOrganizationLearnersFile({ organizationId });
-  });
-  await ApplicationTransaction.execute(async () => {
-    await usecases.saveOrganizationLearnersFile({ organizationId });
-  });
+  await usecases.sendOrganizationLearnersFile({ payload: request.payload, organizationId, userId });
 
   return h.response().code(204);
 };

api/src/prescription/learner-management/domain/models/ImportCommonOrganizationLearnersJob.js

@@ -0,0 +1,5 @@
+export class ImportCommonOrganizationLearnersJob {
+  constructor({ organizationImportId }) {
+    this.organizationImportId = organizationImportId;
+  }
+}

api/src/prescription/learner-management/domain/models/ValidateCommonOrganizationImportFileJob.js

@@ -0,0 +1,5 @@
+export class ValidateCommonOrganizationImportFileJob {
+  constructor({ organizationImportId }) {
+    this.organizationImportId = organizationImportId;
+  }
+}

api/src/prescription/learner-management/domain/usecases/import-from-feature/save-organization-learners-file.js

@@ -4,17 +4,18 @@ import { AggregateImportError } from '../../errors.js';
 import { ImportOrganizationLearnerSet } from '../../models/ImportOrganizationLearnerSet.js';
 
 const saveOrganizationLearnersFile = async function ({
-  organizationId,
+  organizationImportId,
   organizationLearnerImportFormatRepository,
   organizationLearnerRepository,
   organizationImportRepository,
   importStorage,
   dependencies = { getDataBuffer },
 }) {
   const errors = [];
-  const organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
+  const organizationImport = await organizationImportRepository.get(organizationImportId);
 
   try {
+    const organizationId = organizationImport.organizationId;
     const importFormat = await organizationLearnerImportFormatRepository.get(organizationId);
 
     const readableStream = await importStorage.readFile({ filename: organizationImport.filename });

api/src/prescription/learner-management/domain/usecases/import-from-feature/send-organization-learners-file.js

@@ -4,17 +4,19 @@ import { CommonCsvLearnerParser } from '../../../infrastructure/serializers/csv/
 import { getDataBuffer } from '../../../infrastructure/utils/bufferize/get-data-buffer.js';
 import { AggregateImportError, OrganizationLearnerImportFormatNotFoundError } from '../../errors.js';
 import { OrganizationImport } from '../../models/OrganizationImport.js';
+import { ValidateCommonOrganizationImportFileJob } from '../../models/ValidateCommonOrganizationImportFileJob.js';
 
 const sendOrganizationLearnersFile = async function ({
   payload,
   userId,
   organizationId,
   organizationLearnerImportFormatRepository,
+  validateCommonOrganizationImportFileJobRepository,
   organizationImportRepository,
   importStorage,
   dependencies = { createReadStream, getDataBuffer },
 }) {
-  const organizationImport = OrganizationImport.create({ organizationId, createdBy: userId });
+  let organizationImport = OrganizationImport.create({ organizationId, createdBy: userId });
   let filename;
   let encoding;
   const errors = [];
@@ -25,6 +27,10 @@ const sendOrganizationLearnersFile = async function ({
     if (organizationLearnerImportFormat === null)
       throw new OrganizationLearnerImportFormatNotFoundError(organizationId);
 
+    await organizationImportRepository.save(organizationImport);
+
+    organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
+
     const readableStreamEncoding = dependencies.createReadStream(payload.path);
     const bufferEncoding = await dependencies.getDataBuffer(readableStreamEncoding);
 
@@ -36,6 +42,9 @@ const sendOrganizationLearnersFile = async function ({
     encoding = parser.getEncoding();
 
     filename = await importStorage.sendFile({ filepath: payload.path });
+    await validateCommonOrganizationImportFileJobRepository.performAsync(
+      new ValidateCommonOrganizationImportFileJob({ organizationImportId: organizationImport.id }),
+    );
   } catch (error) {
     if (Array.isArray(error)) {
       errors.push(...error);

api/src/prescription/learner-management/domain/usecases/import-from-feature/validate-organization-learners-file.js

@@ -1,19 +1,22 @@
 import { CommonCsvLearnerParser } from '../../../infrastructure/serializers/csv/common-csv-learner-parser.js';
 import { getDataBuffer } from '../../../infrastructure/utils/bufferize/get-data-buffer.js';
 import { AggregateImportError } from '../../errors.js';
+import { ImportCommonOrganizationLearnersJob } from '../../models/ImportCommonOrganizationLearnersJob.js';
 import { ImportOrganizationLearnerSet } from '../../models/ImportOrganizationLearnerSet.js';
 
 const validateOrganizationLearnersFile = async function ({
-  organizationId,
+  organizationImportId,
   organizationLearnerImportFormatRepository,
+  importCommonOrganizationLearnersJobRepository,
   organizationImportRepository,
   importStorage,
   dependencies = { getDataBuffer },
 }) {
   const errors = [];
 
-  const organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
+  const organizationImport = await organizationImportRepository.get(organizationImportId);
   try {
+    const organizationId = organizationImport.organizationId;
     const importFormat = await organizationLearnerImportFormatRepository.get(organizationId);
 
     const readableStream = await importStorage.readFile({ filename: organizationImport.filename });
@@ -29,6 +32,9 @@ const validateOrganizationLearnersFile = async function ({
     });
 
     learnerSet.addLearners(learners);
+    await importCommonOrganizationLearnersJobRepository.performAsync(
+      new ImportCommonOrganizationLearnersJob({ organizationImportId }),
+    );
   } catch (error) {
     if (Array.isArray(error)) {
       errors.push(...error);

api/src/prescription/learner-management/domain/usecases/index.js

@@ -17,9 +17,11 @@ import * as membershipRepository from '../../../../team/infrastructure/repositor
 import * as registrationOrganizationLearnerRepository from '../../../organization-learner/infrastructure/repositories/registration-organization-learner-repository.js';
 import * as campaignParticipationRepository from '../../infrastructure/repositories/campaign-participation-repository.js';
 import { repositories } from '../../infrastructure/repositories/index.js';
+import { importCommonOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-common-organization-learners-job-repository.js';
 import { importOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-organization-learners-job-repository.js';
 import { importScoCsvOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-sco-csv-organization-learners-job-repository.js';
 import { importSupOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-sup-organization-learners-job-repository.js';
+import { validateCommonOrganizationImportFileJobRepository } from '../../infrastructure/repositories/jobs/validate-common-organization-learners-import-file-job-repository.js';
 import { validateCsvOrganizationImportFileJobRepository } from '../../infrastructure/repositories/jobs/validate-csv-organization-learners-import-file-job-repository.js';
 import { validateOrganizationImportFileJobRepository } from '../../infrastructure/repositories/jobs/validate-organization-learners-import-file-job-repository.js';
 import * as organizationImportRepository from '../../infrastructure/repositories/organization-import-repository.js';
@@ -59,6 +61,7 @@ import { importStorage } from '../../infrastructure/storage/import-storage.js';
 const dependencies = {
   campaignParticipationRepository,
   campaignRepository,
+  importCommonOrganizationLearnersJobRepository,
   importOrganizationLearnersJobRepository,
   importScoCsvOrganizationLearnersJobRepository,
   importStorage,
@@ -80,6 +83,7 @@ const dependencies = {
   supOrganizationLearnerRepository,
   userReconciliationService,
   userRepository,
+  validateCommonOrganizationImportFileJobRepository,
   validateCsvOrganizationImportFileJobRepository,
   validateOrganizationImportFileJobRepository,
 };

api/src/prescription/learner-management/infrastructure/repositories/jobs/import-common-organization-learners-job-repository.js

@@ -0,0 +1,18 @@
+import {
+  JobExpireIn,
+  JobRepository,
+  JobRetry,
+} from '../../../../../shared/infrastructure/repositories/jobs/job-repository.js';
+import { ImportCommonOrganizationLearnersJob } from '../../../domain/models/ImportCommonOrganizationLearnersJob.js';
+
+class ImportCommonOrganizationLearnersJobRepository extends JobRepository {
+  constructor() {
+    super({
+      name: ImportCommonOrganizationLearnersJob.name,
+      expireIn: JobExpireIn.HIGH,
+      retry: JobRetry.FEW_RETRY,
+    });
+  }
+}
+
+export const importCommonOrganizationLearnersJobRepository = new ImportCommonOrganizationLearnersJobRepository();

api/src/prescription/learner-management/infrastructure/repositories/jobs/validate-common-organization-learners-import-file-job-repository.js

@@ -0,0 +1,19 @@
+import {
+  JobExpireIn,
+  JobRepository,
+  JobRetry,
+} from '../../../../../shared/infrastructure/repositories/jobs/job-repository.js';
+import { ValidateCommonOrganizationImportFileJob } from '../../../domain/models/ValidateCommonOrganizationImportFileJob.js';
+
+class ValidateCommonOrganizationImportFileJobRepository extends JobRepository {
+  constructor() {
+    super({
+      name: ValidateCommonOrganizationImportFileJob.name,
+      expireIn: JobExpireIn.HIGH,
+      retry: JobRetry.FEW_RETRY,
+    });
+  }
+}
+
+export const validateCommonOrganizationImportFileJobRepository =
+  new ValidateCommonOrganizationImportFileJobRepository();

api/tests/prescription/learner-management/acceptance/application/organization-learners-route_test.js

@@ -1,4 +1,5 @@
 import { IMPORT_KEY_FIELD } from '../../../../../src/prescription/learner-management/domain/constants.js';
+import { getLastByOrganizationId } from '../../../../../src/prescription/learner-management/infrastructure/repositories/organization-import-repository.js';
 import { ORGANIZATION_FEATURE } from '../../../../../src/shared/domain/constants.js';
 import { Membership } from '../../../../../src/shared/domain/models/Membership.js';
 import {
@@ -7,7 +8,6 @@ import {
   expect,
   generateValidRequestAuthorizationHeader,
   insertUserWithRoleSuperAdmin,
-  knex,
 } from '../../../../test-helper.js';
 
 describe('Acceptance | Prescription | learner management | Application | organization-learners-management', function () {
@@ -129,11 +129,11 @@ describe('Acceptance | Prescription | learner management | Application | organiz
       // when
       const response = await server.inject(options);
 
-      const organizationLearners = await knex('organization-learners').where({ organizationId });
+      const organizationImport = await getLastByOrganizationId(organizationId);
       // then
       expect(response.statusCode).to.equal(204);
 
-      expect(organizationLearners).lengthOf(1);
+      expect(organizationImport.status).to.equals('UPLOADED');
     });
   });
 

api/tests/prescription/learner-management/integration/infrastructure/repositories/jobs/import-common-organization-learners-job-repository_test.js

@@ -0,0 +1,27 @@
+import { ImportCommonOrganizationLearnersJob } from '../../../../../../../src/prescription/learner-management/domain/models/ImportCommonOrganizationLearnersJob.js';
+import { importCommonOrganizationLearnersJobRepository } from '../../../../../../../src/prescription/learner-management/infrastructure/repositories/jobs/import-common-organization-learners-job-repository.js';
+import {
+  JobExpireIn,
+  JobRetry,
+} from '../../../../../../../src/shared/infrastructure/repositories/jobs/job-repository.js';
+import { expect } from '../../../../../../test-helper.js';
+
+describe('Integration | Prescription | Infrastructure | Repository | Jobs | importCommonOrganizationLearnersJobRepository', function () {
+  describe('#performAsync', function () {
+    it('publish a job', async function () {
+      // when
+      await importCommonOrganizationLearnersJobRepository.performAsync(
+        new ImportCommonOrganizationLearnersJob({ organizationImportId: 4123132 }),
+      );
+
+      // then
+      await expect(ImportCommonOrganizationLearnersJob.name).to.have.have.been.performed.withJob({
+        expirein: JobExpireIn.HIGH,
+        retrylimit: JobRetry.FEW_RETRY.retryLimit,
+        retrydelay: JobRetry.FEW_RETRY.retryDelay,
+        retrybackoff: JobRetry.FEW_RETRY.retryBackoff,
+        data: { organizationImportId: 4123132 },
+      });
+    });
+  });
+});

api/tests/prescription/learner-management/integration/infrastructure/repositories/jobs/validate-common-organization-learners-import-file-job-repository_test.js

@@ -0,0 +1,27 @@
+import { ValidateCommonOrganizationImportFileJob } from '../../../../../../../src/prescription/learner-management/domain/models/ValidateCommonOrganizationImportFileJob.js';
+import { validateCommonOrganizationImportFileJobRepository } from '../../../../../../../src/prescription/learner-management/infrastructure/repositories/jobs/validate-common-organization-learners-import-file-job-repository.js';
+import {
+  JobExpireIn,
+  JobRetry,
+} from '../../../../../../../src/shared/infrastructure/repositories/jobs/job-repository.js';
+import { expect } from '../../../../../../test-helper.js';
+
+describe('Integration | Prescription | Infrastructure | Repository | Jobs | validateCommonOrganizationImportFileJobRepository', function () {
+  describe('#performAsync', function () {
+    it('publish a job', async function () {
+      // when
+      await validateCommonOrganizationImportFileJobRepository.performAsync(
+        new ValidateCommonOrganizationImportFileJob({ organizationImportId: 4123132 }),
+      );
+
+      // then
+      await expect(ValidateCommonOrganizationImportFileJob.name).to.have.been.performed.withJob({
+        expirein: JobExpireIn.HIGH,
+        retrylimit: JobRetry.FEW_RETRY.retryLimit,
+        retrydelay: JobRetry.FEW_RETRY.retryDelay,
+        retrybackoff: JobRetry.FEW_RETRY.retryBackoff,
+        data: { organizationImportId: 4123132 },
+      });
+    });
+  });
+});