Skip to content

Commit

Permalink
[FEATURE] Rendre asynchrone l'import SUP sur PixOrga (Pix-13792)
Browse files Browse the repository at this point in the history
  • Loading branch information
pix-service-auto-merge authored Nov 20, 2024
2 parents f77d828 + c46bc06 commit 8cc1b6e
Show file tree
Hide file tree
Showing 18 changed files with 404 additions and 100 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { JobController } from '../../../../shared/application/jobs/job-controller.js';
import { config } from '../../../../shared/config.js';
import { DomainError } from '../../../../shared/domain/errors.js';
import { getI18n } from '../../../../shared/infrastructure/i18n/i18n.js';
import { logger as l } from '../../../../shared/infrastructure/utils/logger.js';
import { ImportSupOrganizationLearnersJob } from '../../domain/models/ImportSupOrganizationLearnersJob.js';
import { usecases } from '../../domain/usecases/index.js';

class ImportSupOrganizationLearnersJobController extends JobController {
#logger;

constructor({ logger = l } = {}) {
super(ImportSupOrganizationLearnersJob.name);
this.#logger = logger;
}

get isJobEnabled() {
return config.pgBoss.importFileJobEnabled;
}

async handle({ data }) {
const { organizationImportId, locale, type } = data;

const i18n = getI18n(locale);

try {
if (type === 'ADDITIONAL_STUDENT') {
return await await usecases.importSupOrganizationLearners({
organizationImportId,
i18n,
});
} else if (type === 'REPLACE_STUDENT') {
return await usecases.replaceSupOrganizationLearners({
organizationImportId,
i18n,
});
}
} catch (err) {
if (!(err instanceof DomainError)) {
throw err;
}
this.#logger.error(err);
}
}
}

export { ImportSupOrganizationLearnersJobController };
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const importSupOrganizationLearners = async function (
unlink: fs.unlink,
},
) {
const organizationId = request.params.id;
const organizationId = request.params.organizationId;
const userId = request.auth.credentials.userId;

try {
Expand All @@ -28,10 +28,8 @@ const importSupOrganizationLearners = async function (
Parser: SupOrganizationLearnerParser,
organizationId,
i18n: request.i18n,
});
await usecases.importSupOrganizationLearners({
organizationId,
i18n: request.i18n,
type: 'ADDITIONAL_STUDENT',
performJob: true,
});
} finally {
try {
Expand All @@ -53,7 +51,7 @@ const replaceSupOrganizationLearners = async function (
},
) {
const userId = request.auth.credentials.userId;
const organizationId = request.params.id;
const organizationId = request.params.organizationId;

try {
await usecases.uploadCsvFile({
Expand All @@ -67,10 +65,8 @@ const replaceSupOrganizationLearners = async function (
Parser: SupOrganizationLearnerParser,
organizationId,
i18n: request.i18n,
});
await usecases.replaceSupOrganizationLearners({
organizationId,
i18n: request.i18n,
type: 'REPLACE_STUDENT',
performJob: true,
});
} finally {
// see https://hapi.dev/api/?v=21.3.3#-routeoptionspayloadoutput
Expand All @@ -86,7 +82,7 @@ const replaceSupOrganizationLearners = async function (
};

const getOrganizationLearnersCsvTemplate = async function (request, h, dependencies = { tokenService }) {
const organizationId = request.params.id;
const organizationId = request.params.organizationId;
const token = request.query.accessToken;
const userId = dependencies.tokenService.extractUserId(token);
const template = await usecases.getOrganizationLearnersCsvTemplate({
Expand All @@ -103,7 +99,7 @@ const getOrganizationLearnersCsvTemplate = async function (request, h, dependenc

const updateStudentNumber = async function (request, h) {
const payload = request.payload.data.attributes;
const organizationId = request.params.id;
const organizationId = request.params.organizationId;
const studentNumber = payload['student-number'];
const organizationLearnerId = request.params.organizationLearnerId;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const register = async function (server) {
server.route([
{
method: 'POST',
path: '/api/organizations/{id}/sup-organization-learners/replace-csv',
path: '/api/organizations/{organizationId}/sup-organization-learners/replace-csv',
config: {
pre: [
{
Expand All @@ -30,7 +30,7 @@ const register = async function (server) {
],
validate: {
params: Joi.object({
id: identifiersType.organizationId,
organizationId: identifiersType.organizationId,
}),
},
payload: {
Expand All @@ -57,7 +57,7 @@ const register = async function (server) {
},
{
method: 'POST',
path: '/api/organizations/{id}/sup-organization-learners/import-csv',
path: '/api/organizations/{organizationId}/sup-organization-learners/import-csv',
config: {
pre: [
{
Expand All @@ -67,7 +67,7 @@ const register = async function (server) {
],
validate: {
params: Joi.object({
id: identifiersType.organizationId,
organizationId: identifiersType.organizationId,
}),
},
payload: {
Expand All @@ -94,12 +94,12 @@ const register = async function (server) {
},
{
method: 'GET',
path: '/api/organizations/{id}/organization-learners/csv-template',
path: '/api/organizations/{organizationId}/organization-learners/csv-template',
config: {
auth: false,
validate: {
params: Joi.object({
id: identifiersType.organizationId,
organizationId: identifiersType.organizationId,
}),
query: Joi.object({
accessToken: Joi.string().required(),
Expand All @@ -116,7 +116,7 @@ const register = async function (server) {
},
{
method: 'PATCH',
path: '/api/organizations/{id}/sup-organization-learners/{organizationLearnerId}',
path: '/api/organizations/{organizationId}/sup-organization-learners/{organizationLearnerId}',
config: {
pre: [
{
Expand All @@ -129,7 +129,7 @@ const register = async function (server) {
allowUnknown: true,
},
params: Joi.object({
id: identifiersType.organizationId,
organizationId: identifiersType.organizationId,
organizationLearnerId: identifiersType.organizationLearnerId,
}),
payload: Joi.object({
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export class ImportSupOrganizationLearnersJob {
constructor({ organizationImportId, type, locale }) {
this.organizationImportId = organizationImportId;
this.type = type;
this.locale = locale;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@ import { getDataBuffer } from '../../infrastructure/utils/bufferize/get-data-buf
import { AggregateImportError } from '../errors.js';

const importSupOrganizationLearners = async function ({
organizationId,
organizationImportId,
i18n,
supOrganizationLearnerRepository,
organizationImportRepository,
importStorage,
}) {
const organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
const organizationImport = await organizationImportRepository.get(organizationImportId);
const errors = [];

// Reading File
try {
const readableStream = await importStorage.readFile({ filename: organizationImport.filename });

const buffer = await getDataBuffer(readableStream);
const parser = SupOrganizationLearnerParser.buildParser(buffer, organizationId, i18n);
const parser = SupOrganizationLearnerParser.buildParser(buffer, organizationImport.organizationId, i18n);

const { learners } = parser.parse(parser.getFileEncoding());

Expand Down
36 changes: 21 additions & 15 deletions api/src/prescription/learner-management/domain/usecases/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import * as membershipRepository from '../../../../team/infrastructure/repositor
import * as campaignParticipationRepository from '../../infrastructure/repositories/campaign-participation-repository.js';
import { repositories } from '../../infrastructure/repositories/index.js';
import { importOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-organization-learners-job-repository.js';
import { importSupOrganizationLearnersJobRepository } from '../../infrastructure/repositories/jobs/import-sup-organization-learners-job-repository.js';
import { validateOrganizationImportFileJobRepository } from '../../infrastructure/repositories/jobs/validate-organization-learners-import-file-job-repository.js';
import * as organizationImportRepository from '../../infrastructure/repositories/organization-import-repository.js';
import * as organizationLearnerImportFormatRepository from '../../infrastructure/repositories/organization-learner-import-format-repository.js';
Expand All @@ -21,37 +22,42 @@ import * as supOrganizationLearnerRepository from '../../infrastructure/reposito
import { importStorage } from '../../infrastructure/storage/import-storage.js';

/**
* @typedef {import ('../../../../../lib/infrastructure/repositories/campaign-repository.js')} CampaignRepository
* @typedef {import ('../../infrastructure/repositories/organization-feature-repository.js')} CampaignParticipationRepository
* @typedef {import ('../../../../../lib/infrastructure/repositories/campaign-repository.js')} CampaignRepository
* @typedef {import ('../../infrastructure/repositories/jobs/import-organization-learners-job-repository.js')} ImportOrganizationLearnersJobRepository
* @typedef {import ('../../infrastructure/storage/import-storage.js')} ImportStorage
* @typedef {import ('../../infrastructure/repositories/jobs/import-sup-organization-learners-job-repository.js')} ImportSupOrganizationLearnersJobRepository
* @typedef {import ('../../../../shared/infrastructure/monitoring-tools.js')} LogErrorWithCorrelationIds
* @typedef {import ('../../../../shared/infrastructure/utils/logger.js')} Loggger
* @typedef {import ('../../../../team/infrastructure/repositories/membership-repository.js')} MembershipRepository
* @typedef {import ('../../infrastructure/repositories/organization-learner-repository.js')} OrganizationLearnerRepository
* @typedef {import ('../../../../organizational-entities/application/api/organization-features-api.js')} OrganizationFeatureApi
* @typedef {import ('../../infrastructure/repositories/organization-feature-repository.js')} OrganizationFeatureRepository
* @typedef {import ('../../infrastructure/repositories/organization-import-repository.js')} OrganizationImportRepository
* @typedef {import ('../../infrastructure/repositories/organization-learner-import-format-repository.js')} OrganizationLearnerImportFormatRepository
* @typedef {import ('../../infrastructure/repositories/organization-learner-repository.js')} OrganizationLearnerRepository
* @typedef {import ('../../../../shared/infrastructure/repositories/organization-repository.js')} OrganizationRepository
* @typedef {import ('../../infrastructure/repositories/organization-import-repository.js')} OrganizationImportRepository
* @typedef {import ('../../infrastructure/repositories/sup-organization-learner-repository.js')} SupOrganizationLearnerRepository
* @typedef {import ('../../../../organizational-entities/application/api/organization-features-api.js')} OrganizationFeatureApi
* @typedef {import ('../../../../../src/shared/infrastructure/utils/logger.js')} logger
* @typedef {import ('../../../../../lib/domain/services/user-reconciliation-service.js')} UserReconciliationService
* @typedef {import ('../../infrastructure/repositories/organization-feature-repository.js')} OrganizationFeatureRepository
* @typedef {import ('../../infrastructure/repositories/jobs/validate-organization-learners-import-file-job-repository.js')} ValidateOrganizationImportFileJobRepository
*/
const dependencies = {
campaignRepository,
campaignParticipationRepository,
campaignRepository,
importOrganizationLearnersJobRepository,
importStorage,
importSupOrganizationLearnersJobRepository,
logErrorWithCorrelationIds,
logger,
membershipRepository,
organizationLearnerRepository,
organizationFeatureApi,
organizationFeatureRepository: repositories.organizationFeatureRepository,
organizationImportRepository,
organizationLearnerImportFormatRepository,
organizationLearnerRepository,
organizationRepository,
importOrganizationLearnersJobRepository,
validateOrganizationImportFileJobRepository,
organizationImportRepository,
supOrganizationLearnerRepository,
organizationFeatureApi,
logErrorWithCorrelationIds,
userReconciliationService,
organizationFeatureRepository: repositories.organizationFeatureRepository,
logger,
validateOrganizationImportFileJobRepository,
};

const path = dirname(fileURLToPath(import.meta.url));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,27 @@ import { getDataBuffer } from '../../infrastructure/utils/bufferize/get-data-buf
import { AggregateImportError } from '../errors.js';

const replaceSupOrganizationLearners = async function ({
organizationId,
organizationImportId,
i18n,
supOrganizationLearnerRepository,
organizationImportRepository,
importStorage,
}) {
const errors = [];
const organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
const organizationImport = await organizationImportRepository.get(organizationImportId);

try {
const readableStream = await importStorage.readFile({ filename: organizationImport.filename });
const buffer = await getDataBuffer(readableStream);
const parser = SupOrganizationLearnerParser.buildParser(buffer, organizationId, i18n);
const parser = SupOrganizationLearnerParser.buildParser(buffer, organizationImport.organizationId, i18n);

const { learners } = parser.parse(parser.getFileEncoding());

await supOrganizationLearnerRepository.replaceStudents(organizationId, learners, organizationImport.createdBy);
await supOrganizationLearnerRepository.replaceStudents(
organizationImport.organizationId,
learners,
organizationImport.createdBy,
);
} catch (error) {
if (error instanceof AggregateImportError) {
errors.push(...error.meta);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
import { AggregateImportError } from '../errors.js';
import { ImportSupOrganizationLearnersJob } from '../models/ImportSupOrganizationLearnersJob.js';

const validateCsvFile = async function ({ Parser, organizationId, i18n, organizationImportRepository, importStorage }) {
const validateCsvFile = async function ({
Parser,
organizationId,
i18n,
type,
performJob,
importSupOrganizationLearnersJobRepository,
organizationImportRepository,
importStorage,
}) {
const organizationImport = await organizationImportRepository.getLastByOrganizationId(organizationId);
const errors = [];
let warningsData;
Expand All @@ -15,12 +25,25 @@ const validateCsvFile = async function ({ Parser, organizationId, i18n, organiza
const { warnings } = parser.parse(parser.getFileEncoding());

warningsData = warnings;

if (performJob) {
await importSupOrganizationLearnersJobRepository.performAsync(
new ImportSupOrganizationLearnersJob({
organizationImportId: organizationImport.id,
type,
locale: i18n.getLocale(),
}),
);
}
} catch (error) {
if (error instanceof AggregateImportError) {
errors.push(...error.meta);
} else {
errors.push(error);
}

await importStorage.deleteFile({ filename: organizationImport.filename });

throw error;
} finally {
organizationImport.validate({ errors, warnings: warningsData });
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import {
JobExpireIn,
JobRepository,
JobRetry,
} from '../../../../../shared/infrastructure/repositories/jobs/job-repository.js';
import { ImportSupOrganizationLearnersJob } from '../../../domain/models/ImportSupOrganizationLearnersJob.js';

class ImportSupOrganizationLearnersJobRepository extends JobRepository {
constructor() {
super({
name: ImportSupOrganizationLearnersJob.name,
expireIn: JobExpireIn.HIGH,
retry: JobRetry.FEW_RETRY,
});
}
}

export const importSupOrganizationLearnersJobRepository = new ImportSupOrganizationLearnersJobRepository();
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { knex } from '../../../../../db/knex-database-connection.js';
import { DomainTransaction } from '../../../../shared/domain/DomainTransaction.js';
import { ApplicationTransaction } from '../../../shared/infrastructure/ApplicationTransaction.js';
import { OrganizationImport } from '../../domain/models/OrganizationImport.js';
import { OrganizationImportDetail } from '../../domain/read-models/OrganizationImportDetail.js';

Expand Down Expand Up @@ -31,7 +30,7 @@ const getLastImportDetailForOrganization = async function (organizationId) {
};

const get = async function (id) {
const knexConn = ApplicationTransaction.getConnection();
const knexConn = DomainTransaction.getConnection();
const result = await knexConn('organization-imports').where({ id }).first();

if (!result) return null;
Expand Down
Loading

0 comments on commit 8cc1b6e

Please sign in to comment.