[FEATURE] Permettre à partir d'un fichier de configuration de parser …

…un CSV (PIX-11614)

 #8415

api/src/prescription/learner-management/domain/models/CommonOrganizationLearnerSet.js

@@ -0,0 +1,168 @@
+import { CsvImportError, ModelValidationError } from '../../../../shared/domain/errors.js';
+import { convertDateValue } from '../../../../shared/infrastructure/utils/date-utils.js';
+import { validateCommonOrganizationLearner } from '../validators/common-organization-learner-validator.js';
+
+class ImportOrganizationLearnerSet {
+  #learners;
+  #hasValidationFormats;
+  #hasUnicityRules;
+  #unicityKeys;
+  #errors;
+  #organizationId;
+  #columnMapping;
+
+  constructor({ organizationId, validationRules = {}, columnMapping }) {
+    this.#organizationId = organizationId;
+    this.#learners = [];
+    this.validationRules = validationRules;
+    this.#hasUnicityRules = !!validationRules?.unicity;
+    this.#hasValidationFormats = !!validationRules?.formats;
+    this.#columnMapping = columnMapping;
+    this.#unicityKeys = [];
+    this.#errors = [];
+  }
+
+  #lineToOrganizationLearnerAttributes(learner) {
+    const learnerAttributes = {
+      organizationId: this.#organizationId,
+    };
+
+    this.#columnMapping.forEach((column) => {
+      const value = learner[column.name];
+      if (column.property) {
+        learnerAttributes[column.property] = value;
+      } else {
+        learnerAttributes[column.name] = this.#formatLearnerAttribute({ attribute: value, columnName: column.name });
+      }
+    });
+
+    return learnerAttributes;
+  }
+
+  #formatLearnerAttribute({ attribute, columnName }) {
+    if (!attribute) return null;
+
+    if (this.validationRules.formats) {
+      const dateFormat = this.validationRules.formats.find((rule) => rule.type === 'date' && rule.name === columnName);
+
+      if (dateFormat) {
+        return convertDateValue({
+          dateString: attribute,
+          inputFormat: dateFormat.format,
+          alternativeInputFormat: dateFormat.format,
+          outputFormat: 'YYYY-MM-DD',
+        });
+      }
+    }
+
+    return attribute.toString();
+  }
+
+  addLearners(learners) {
+    learners.forEach((learner, index) => {
+      try {
+        this.#validateRules(learner);
+        const commonOrganizationLearner = new CommonOrganizationLearner(
+          this.#lineToOrganizationLearnerAttributes(learner),
+        );
+        this.#learners.push(commonOrganizationLearner);
+      } catch (errors) {
+        this.#handleValidationError(errors, index);
+      }
+    });
+
+    if (this.#errors.length > 0) {
+      throw this.#errors;
+    }
+  }
+
+  #handleValidationError(errors, index) {
+    errors.forEach((error) => {
+      const line = this.#getCsvLine(index);
+      const field = error.key;
+
+      if (error.why === 'uniqueness') {
+        this.#errors.push(new CsvImportError(error.code, { line, field }));
+      }
+
+      if (error.why === 'date_format') {
+        this.#errors.push(new CsvImportError(error.code, { line, field, acceptedFormat: error.acceptedFormat }));
+      }
+
+      if (error.why === 'field_required') {
+        this.#errors.push(new CsvImportError(error.code, { line, field }));
+      }
+    });
+  }
+
+  #getCsvLine(index) {
+    const LEARNER_DATA_CSV_STARTING_AT_LINE = 2;
+
+    return index + LEARNER_DATA_CSV_STARTING_AT_LINE;
+  }
+
+  #checkUnicityRule(learner) {
+    const learnerUnicityValues = this.#getLearnerUnicityValues(learner);
+    if (!this.#unicityKeys.includes(learnerUnicityValues)) {
+      this.#unicityKeys.push(learnerUnicityValues);
+      return null;
+    } else {
+      return ModelValidationError.unicityError({
+        key: this.validationRules.unicity.join('-'),
+      });
+    }
+  }
+
+  #getLearnerUnicityValues(learner) {
+    const unicityKeys = [];
+    this.validationRules.unicity.forEach((rule) => {
+      unicityKeys.push(learner[rule]);
+    });
+    return unicityKeys.join('-');
+  }
+
+  #validateRules(learner) {
+    const errors = [];
+
+    if (this.#hasUnicityRules) {
+      const unicityError = this.#checkUnicityRule(learner);
+
+      if (unicityError) {
+        errors.push(unicityError);
+      }
+    }
+
+    if (this.#hasValidationFormats) {
+      const validationErrors = this.#checkValidations(learner);
+
+      if (validationErrors) {
+        errors.push(...validationErrors);
+      }
+    }
+
+    if (errors.length > 0) {
+      throw errors;
+    }
+  }
+
+  #checkValidations(learner) {
+    return validateCommonOrganizationLearner(learner, this.validationRules.formats);
+  }
+
+  get learners() {
+    return this.#learners;
+  }
+}
+
+class CommonOrganizationLearner {
+  constructor({ id, userId, lastName, firstName, organizationId, ...attributes } = {}) {
+    if (id) this.id = id;
+    if (userId) this.userId = userId;
+    this.lastName = lastName;
+    this.firstName = firstName;
+    this.organizationId = organizationId;
+    if (attributes) this.attributes = attributes;
+  }
+}
+
+export { CommonOrganizationLearner, ImportOrganizationLearnerSet };

api/src/prescription/learner-management/domain/validators/common-organization-learner-validator.js

@@ -0,0 +1,39 @@
+import JoiDate from '@joi/date';
+import BaseJoi from 'joi';
+
+import { ModelValidationError } from '../../../../shared/domain/errors.js';
+const Joi = BaseJoi.extend(JoiDate);
+
+const validationConfiguration = { allowUnknown: true, abortEarly: false };
+
+const validateCommonOrganizationLearner = function (commonOrganizationLearner, validationFormatRules) {
+  const customAttributeRule = {};
+  validationFormatRules?.forEach(({ name, type, format, required, min, max }) => {
+    if (type === 'date') {
+      customAttributeRule[name] = Joi.date()
+        .format(format)
+        .presence(required ? 'required' : 'optional');
+    }
+
+    if (type === 'string') {
+      customAttributeRule[name] = Joi.string()
+        .min(min || 0)
+        .max(max || 255)
+        .presence(required ? 'required' : 'optional');
+    }
+  });
+  const validationSchema = Joi.object({
+    ...customAttributeRule,
+  });
+
+  const { error: validationErrors } = validationSchema.validate(commonOrganizationLearner, validationConfiguration);
+  if (validationErrors?.details.length > 0) {
+    return validationErrors.details.map((error) => {
+      return ModelValidationError.fromJoiError(error);
+    });
+  } else {
+    return [];
+  }
+};
+
+export { validateCommonOrganizationLearner };

api/src/prescription/learner-management/infrastructure/serializers/csv/common-csv-learner-parser.js

@@ -0,0 +1,121 @@
+import iconv from 'iconv-lite';
+import papa from 'papaparse';
+
+import { CsvImportError } from '../../../../../shared/domain/errors.js';
+
+const ERRORS = {
+  ENCODING_NOT_SUPPORTED: 'ENCODING_NOT_SUPPORTED',
+  BAD_CSV_FORMAT: 'BAD_CSV_FORMAT',
+  HEADER_REQUIRED: 'HEADER_REQUIRED',
+  HEADER_UNKNOWN: 'HEADER_UNKNOWN',
+};
+
+const PARSING_OPTIONS = {
+  header: true,
+  skipEmptyLines: 'greedy',
+  transform: (value) => {
+    if (typeof value === 'string') {
+      value = value.trim();
+      return value.length ? value : undefined;
+    }
+    return value;
+  },
+};
+
+class CommonCsvLearnerParser {
+  #input;
+  #errors;
+
+  // compute heading
+  #columns;
+  // compute support_enconding
+  #supportedEncodings;
+
+  constructor(input, headingConfiguration, encodingConfiguration) {
+    this.#input = input;
+    this.#errors = [];
+
+    // compute heading
+    this.#columns = headingConfiguration;
+
+    // compute support_enconding
+    this.#supportedEncodings = encodingConfiguration;
+  }
+
+  parse(encoding) {
+    const { learnerLines, fields } = this.#parse(encoding);
+
+    this.#throwHasErrors();
+
+    this.#checkColumns(fields);
+
+    this.#throwHasErrors();
+
+    return learnerLines;
+  }
+
+  /**
+   * Identify which encoding has the given file.
+   * To check it, we decode and parse the first line of the file with supported encodings.
+   * If there is one with at least "First name" or "Student number" correctly parsed and decoded.
+   */
+  getEncoding() {
+    const supported_encodings = this.#supportedEncodings;
+    for (const encoding of supported_encodings) {
+      const decodedInput = iconv.decode(this.#input, encoding);
+      if (!decodedInput.includes('�')) {
+        return encoding;
+      }
+    }
+
+    this.#errors.push(new CsvImportError(ERRORS.ENCODING_NOT_SUPPORTED));
+    this.#throwHasErrors();
+  }
+
+  #throwHasErrors() {
+    if (this.#errors.length > 0) throw this.#errors;
+  }
+
+  #parse(encoding) {
+    const decodedInput = iconv.decode(this.#input, encoding);
+    const {
+      data: learnerLines,
+      meta: { fields },
+      errors,
+    } = papa.parse(decodedInput, PARSING_OPTIONS);
+
+    if (errors.length) {
+      const hasErrors = errors.some((error) => ['Delimiter', 'FieldMismatch'].includes(error.type));
+
+      if (hasErrors) {
+        this.#errors.push(new CsvImportError(ERRORS.BAD_CSV_FORMAT));
+      }
+    }
+
+    this.#throwHasErrors();
+
+    return { learnerLines, fields };
+  }
+
+  #checkColumns(parsedColumns) {
+    // Required columns
+    const mandatoryColumn = this.#columns.filter((c) => c.isRequired);
+
+    mandatoryColumn.forEach((colum) => {
+      if (!parsedColumns.includes(colum.name)) {
+        this.#errors.push(new CsvImportError(ERRORS.HEADER_REQUIRED, { field: colum.name }));
+      }
+    });
+
+    // Expected columns
+    const acceptedColumns = this.#columns.map((column) => column.name);
+
+    const unknowColumns = parsedColumns.filter((columnName) => !acceptedColumns.includes(columnName));
+
+    unknowColumns.forEach((columnName) => {
+      if (columnName !== '') this.#errors.push(new CsvImportError(ERRORS.HEADER_UNKNOWN, { field: columnName }));
+    });
+  }
+}
+
+export { CommonCsvLearnerParser };

api/src/shared/domain/constants.js

@@ -32,9 +32,13 @@ const ORGANIZATION_FEATURE = {
   },
 };
 
-const constants = {
-  LEVENSHTEIN_DISTANCE_MAX_RATE,
-  ORGANIZATION_FEATURE,
+const VALIDATION_ERRORS = {
+  PROPERTY_NOT_UNIQ: 'PROPERTY_NOT_UNIQ',
+  FIELD_DATE_FORMAT: 'FIELD_DATE_FORMAT',
+  FIELD_REQUIRED: 'FIELD_REQUIRED',
+  FIELD_NOT_STRING: 'FIELD_NOT_STRING',
+  FIELD_STRING_MIN: 'FIELD_STRING_MIN',
+  FIELD_STRING_MAX: 'FIELD_STRING_MAX',
 };
 
-export { constants, LEVENSHTEIN_DISTANCE_MAX_RATE, LOCALE, ORGANIZATION_FEATURE, SUPPORTED_LOCALES };
+export { LEVENSHTEIN_DISTANCE_MAX_RATE, LOCALE, ORGANIZATION_FEATURE, SUPPORTED_LOCALES, VALIDATION_ERRORS };