From b5473be405071923fd0c11db9873f7717f79e722 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 15 May 2024 22:00:48 +0300 Subject: [PATCH 001/229] AMP-30858- Create Data importer for txt --- .../module/aim/action/DataImporter.java | 26 ++++--- amp/repository/aim/view/dataImporter.jsp | 74 ++++++++++++++++--- 2 files changed, 79 insertions(+), 21 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index 06fda8b2a5c..673849f35e9 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -68,20 +68,23 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet if (request.getParameter("uploadTemplate")!=null) { logger.info(" this is the action "+request.getParameter("uploadTemplate")); + Set headersSet = new HashSet<>(); InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - Set headersSet = new HashSet<>(); - for (int i=0;i cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); + if (request.getParameter("fileType")!=null && (Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { + Workbook workbook = new XSSFWorkbook(fileInputStream); + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + Row headerRow = sheet.getRow(0); + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + headersSet.add(cell.getStringCellValue()); + } + } + workbook.close(); } headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); @@ -97,7 +100,6 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet dataImporterForm.getColumnPairs().clear(); - workbook.close(); } diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index d77b9382552..6f5d68a1378 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -15,7 +15,7 @@ // If the search string is not found, return the original string return inputString; } else { - // Construct the new string with the replacement + // Construct the new strin8klg with the replacement return inputString.substring(0, lastIndex) + replacement + inputString.substring(lastIndex + search.length); } } @@ -36,6 +36,34 @@ sendValuesToBackend(columnName,selectedField,"removeField"); }); + $('#file-type').change(function() { + var fileType = $(this).val(); + console.log("File type selected: " + fileType==='excel'); + if (fileType === "csv") { + $('#select-file-label').html("Select csv file"); + $('#data-file').attr("accept", ".csv"); + $('#template-file').attr("accept", ".csv"); + $('#separator-div').hide(); + + } else if(fileType==="text") { + $('#select-file-label').html("Select text file"); + $('#data-file').attr("accept", ".txt"); + $('#template-file').attr("accept", ".txt"); + $('#separator-div').show(); + } + else if(fileType==="excel") { + $('#select-file-label').html("Select excel file"); + $('#data-file').attr("accept", ".xls,.xlsx"); + $('#template-file').attr("accept", ".xls,.xlsx"); + $('#separator-div').hide(); + } + else if(fileType==="json") { + $('#select-file-label').html("Select json file"); + $('#data-file').attr("accept", ".json"); + $('#template-file').attr("accept", ".json"); + $('#separator-div').hide(); + } + }) }); function sendValuesToBackend(columnName, selectedField, action) @@ -117,8 +145,13 @@ function uploadTemplateFile() { var formData = new FormData(); var fileInput = document.getElementById('templateFile'); + var fileType = $('#file-type').val(); + var dataSeparator = $('#data-separator').val(); + formData.append('templateFile', fileInput.files[0]); formData.append('uploadTemplate', "uploadTemplate"); + formData.append('fileType', fileType); + formData.append('dataSeparator', dataSeparator); var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); @@ -143,6 +176,9 @@ function uploadDataFile() { var formData = new FormData(); var fileInput = document.getElementById('dataFile'); + var fileType = $('#file-type').val(); + var dataSeparator = $('#data-separator').val(); + // Check if a file is selected if (!fileInput.files.length) { alert("Please select a file to upload."); @@ -150,6 +186,8 @@ } formData.append('dataFile', fileInput.files[0]); formData.append('uploadDataFile',"uploadDataFile"); + formData.append('fileType',fileType); + formData.append('dataSeparator',dataSeparator); var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); @@ -193,9 +231,31 @@

Upload File

Data file configuration

+ +
+ + +
- + +
+

@@ -204,11 +264,6 @@ - -<%-- --%> -<%-- --%> - -<%-- --%>

@@ -242,9 +297,10 @@

- + + <%-- --%> - +

<%-- Upload--%> From 8fbacf30da247437470762624e69b880645a2dad Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 15 May 2024 22:11:47 +0300 Subject: [PATCH 002/229] AMP-30858- Create Data importer for txt --- amp/repository/aim/view/dataImporter.jsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index 6f5d68a1378..e29ffe49ec1 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -237,9 +237,9 @@
From 25ddcb77a4a7e8e540b49cf0433811d2b836ee0e Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 15 May 2024 22:19:55 +0300 Subject: [PATCH 003/229] AMP-30858- Create Data importer for txt --- amp/repository/aim/view/dataImporter.jsp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index e29ffe49ec1..c0093aaa821 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -248,9 +248,9 @@
From e7ab95ac454566b16d3b6c4847176913b184261a Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 16 May 2024 13:35:36 +0300 Subject: [PATCH 004/229] AMP-30858- Create Data importer for txt --- .../module/aim/action/DataImporter.java | 49 ++++++++++++++----- amp/pom.xml | 7 +++ amp/repository/aim/view/dataImporter.jsp | 4 +- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index 673849f35e9..33bb67687bb 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -4,6 +4,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; import org.apache.poi.ss.usermodel.Cell; @@ -70,22 +75,40 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info(" this is the action "+request.getParameter("uploadTemplate")); Set headersSet = new HashSet<>(); - InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - if (request.getParameter("fileType")!=null && (Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - for (int i = 0; i < numberOfSheets; i++) { - Sheet sheet = workbook.getSheetAt(i); - Row headerRow = sheet.getRow(0); - Iterator cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); + if (request.getParameter("fileType")!=null) { + InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); + if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { + Workbook workbook = new XSSFWorkbook(fileInputStream); + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + Row headerRow = sheet.getRow(0); + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + headersSet.add(cell.getStringCellValue()); + } + } + workbook.close(); - } - workbook.close(); + } else if (Objects.equals(request.getParameter("fileType"), "text")) { + CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); + + try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { + String[] headers = reader.readNext(); // Read the first line which contains headers + if (headers != null) { + // Print each header + headersSet.addAll(Arrays.asList(headers)); + } else { + logger.info("File is empty or does not contain headers."); + } + } catch (IOException | CsvValidationException e) { + e.printStackTrace(); + } + + } } headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); StringBuilder headers = new StringBuilder(); diff --git a/amp/pom.xml b/amp/pom.xml index 8883d3ee5f4..e0ee3798b99 100644 --- a/amp/pom.xml +++ b/amp/pom.xml @@ -222,6 +222,13 @@ dom4j 1.6.1 + + + com.opencsv + opencsv + 5.5.2 + + net.sf.ehcache ehcache-core diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index c0093aaa821..efcb4cd46d2 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -255,7 +255,7 @@
- +

@@ -300,7 +300,7 @@ <%-- --%> - +

<%-- Upload--%> From a726c70df641ce6829949d5bde35b0afb61140cd Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 20 May 2024 11:43:03 +0300 Subject: [PATCH 005/229] AMP-30858- Create txt importer --- .../module/aim/action/DataImporter.java | 289 +++++++++--------- 1 file changed, 144 insertions(+), 145 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index aae8d46900b..ec133383dde 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -60,7 +60,7 @@ public class DataImporter extends Action { static Logger logger = LoggerFactory.getLogger(DataImporter.class); private static final int BATCH_SIZE = 1000; - private static Map constantsMap=new HashMap<>(); + private static Map constantsMap = new HashMap<>(); @Override @@ -68,191 +68,190 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet // List of fields List fieldsInfo = getEntityFieldsInfo(); request.setAttribute("fieldsInfo", fieldsInfo); - DataImporterForm dataImporterForm = (DataImporterForm)form; + DataImporterForm dataImporterForm = (DataImporterForm) form; if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { - logger.info(" this is the action "+request.getParameter("action")); - if (request.getParameter("uploadTemplate")!=null) { - logger.info(" this is the action "+request.getParameter("uploadTemplate")); - Set headersSet = new HashSet<>(); - - if (request.getParameter("fileType")!=null) { - InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - for (int i = 0; i < numberOfSheets; i++) { - Sheet sheet = workbook.getSheetAt(i); - Row headerRow = sheet.getRow(0); - Iterator cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); - } - - } - workbook.close(); + logger.info(" this is the action " + request.getParameter("action")); + if (request.getParameter("uploadTemplate") != null) { + logger.info(" this is the action " + request.getParameter("uploadTemplate")); + Set headersSet = new HashSet<>(); + + if (request.getParameter("fileType") != null) { + InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); + if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { + Workbook workbook = new XSSFWorkbook(fileInputStream); + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + Row headerRow = sheet.getRow(0); + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + headersSet.add(cell.getStringCellValue()); + } - } else if (Objects.equals(request.getParameter("fileType"), "text")) { - CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); - - try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { - String[] headers = reader.readNext(); // Read the first line which contains headers - - if (headers != null) { - // Print each header - headersSet.addAll(Arrays.asList(headers)); - } else { - logger.info("File is empty or does not contain headers."); } - } catch (IOException | CsvValidationException e) { - e.printStackTrace(); - } + workbook.close(); + + } else if (Objects.equals(request.getParameter("fileType"), "text")) { + CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); + + try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { + String[] headers = reader.readNext(); // Read the first line which contains headers + + if (headers != null) { + // Print each header + headersSet.addAll(Arrays.asList(headers)); + } else { + logger.info("File is empty or does not contain headers."); + } + } catch (IOException | CsvValidationException e) { + e.printStackTrace(); + } + } } - } - headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); - StringBuilder headers = new StringBuilder(); - headers.append(" \n"); - response.setHeader("selectTag",headers.toString()); + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setHeader("selectTag", headers.toString()); - response.setHeader("updatedMap",""); + response.setHeader("updatedMap", ""); - dataImporterForm.getColumnPairs().clear(); + dataImporterForm.getColumnPairs().clear(); } - - if (Objects.equals(request.getParameter("action"), "addField")) { - logger.info(" this is the action "+request.getParameter("action")); + logger.info(" this is the action " + request.getParameter("action")); - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - logger.info("Column Pairs:"+dataImporterForm.getColumnPairs()); + String columnName = request.getParameter("columnName"); + String selectedField = request.getParameter("selectedField"); + dataImporterForm.getColumnPairs().put(columnName, selectedField); + logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString( dataImporterForm.getColumnPairs()); + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); - // Send JSON response - response.setContentType("application/json"); - response.getWriter().write(json); - response.setCharacterEncoding("UTF-8"); + // Send JSON response + response.setContentType("application/json"); + response.getWriter().write(json); + response.setCharacterEncoding("UTF-8"); return null; - } + } - if (Objects.equals(request.getParameter("action"), "removeField")) { - logger.info(" this is the action "+request.getParameter("action")); + if (Objects.equals(request.getParameter("action"), "removeField")) { + logger.info(" this is the action " + request.getParameter("action")); - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - removeMapItem(dataImporterForm.getColumnPairs(),columnName,selectedField); - logger.info("Column Pairs:"+dataImporterForm.getColumnPairs()); + String columnName = request.getParameter("columnName"); + String selectedField = request.getParameter("selectedField"); + dataImporterForm.getColumnPairs().put(columnName, selectedField); + removeMapItem(dataImporterForm.getColumnPairs(), columnName, selectedField); + logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString( dataImporterForm.getColumnPairs()); + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); - // Send JSON response - response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); + // Send JSON response + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); // response.setHeader("updatedMap",json); - response.getWriter().write(json); + response.getWriter().write(json); - return null; - - } + return null; - if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { - logger.info("This is the action Upload " + request.getParameter("uploadDataFile")); - String fileName = dataImporterForm.getDataFile().getFileName(); - String tempDirPath = System.getProperty("java.io.tmpdir"); - File tempDir = new File(tempDirPath); - if (!tempDir.exists()) { - tempDir.mkdirs(); } - String tempFilePath = tempDirPath + File.separator + fileName; - try (InputStream inputStream = dataImporterForm.getDataFile().getInputStream(); - FileOutputStream outputStream = new FileOutputStream(tempFilePath)) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); + + if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { + logger.info("This is the action Upload " + request.getParameter("uploadDataFile")); + String fileName = dataImporterForm.getDataFile().getFileName(); + String tempDirPath = System.getProperty("java.io.tmpdir"); + File tempDir = new File(tempDirPath); + if (!tempDir.exists()) { + tempDir.mkdirs(); + } + String tempFilePath = tempDirPath + File.separator + fileName; + try (InputStream inputStream = dataImporterForm.getDataFile().getInputStream(); + FileOutputStream outputStream = new FileOutputStream(tempFilePath)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } } - } - // Check if the file is readable and has correct content - File tempFile = new File(tempFilePath); - List similarFiles = ImportedFileUtil.getSimilarFiles(tempFile); - if (similarFiles!=null && !similarFiles.isEmpty()) { - for (ImportedFilesRecord similarFilesRecord : similarFiles) { - logger.info("Similar file: " + similarFilesRecord); - if (similarFilesRecord.getImportStatus().equals(ImportStatus.IN_PROGRESS)) - { - response.setHeader("errorMessage","You have a similar file in progress. Please try again later."); + // Check if the file is readable and has correct content + File tempFile = new File(tempFilePath); + List similarFiles = ImportedFileUtil.getSimilarFiles(tempFile); + if (similarFiles != null && !similarFiles.isEmpty()) { + for (ImportedFilesRecord similarFilesRecord : similarFiles) { + logger.info("Similar file: " + similarFilesRecord); + if (similarFilesRecord.getImportStatus().equals(ImportStatus.IN_PROGRESS)) { + response.setHeader("errorMessage", "You have a similar file in progress. Please try again later."); + response.setStatus(400); + return mapping.findForward("importData"); + } + } + } + if (dataImporterForm.getColumnPairs().isEmpty() || !dataImporterForm.getColumnPairs().containsValue("{projectTitle}")) { + response.setHeader("errorMessage", "You must have atleast the {projectTitle} key in your config."); response.setStatus(400); return mapping.findForward("importData"); } - } - } - if (dataImporterForm.getColumnPairs().isEmpty() || !dataImporterForm.getColumnPairs().containsValue("{projectTitle}")) - { - response.setHeader("errorMessage","You must have atleast the {projectTitle} key in your config."); - response.setStatus(400); - return mapping.findForward("importData"); - } - if (!isFileReadable(tempFile) || !isFileContentValid(tempFile)) { - // Handle invalid file - logger.error("Invalid file or content."); - response.setHeader("errorMessage","Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); - - // Optionally, you can respond with an error message to the client. - } else { - // Proceed with processing the file - ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); - // Process the file in batches - int res = processFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); - if (res != 1) { - // Handle error - logger.info("Error processing file " + tempFile); - response.setHeader("errorMessage","Unable to parse the file. Please check the file format/content and try again."); + if (!isFileReadable(tempFile) || !isFileContentValid(tempFile)) { + // Handle invalid file + logger.error("Invalid file or content."); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); response.setStatus(400); return mapping.findForward("importData"); - } - // Clean up - Files.delete(tempFile.toPath()); - logger.info("Cache map size: " + constantsMap.size()); - constantsMap.clear(); - logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); - Instant start = Instant.now(); - logger.info("Start time: " + start); - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - logger.info("Time Elapsed: " + timeElapsed); - - // Send response - response.setHeader("updatedMap", ""); - dataImporterForm.getColumnPairs().clear(); + // Optionally, you can respond with an error message to the client. + } else { + // Proceed with processing the file + ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); + // Process the file in batches + int res = processFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); + if (res != 1) { + // Handle error + logger.info("Error processing file " + tempFile); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + response.setStatus(400); + return mapping.findForward("importData"); + } + + // Clean up + Files.delete(tempFile.toPath()); + logger.info("Cache map size: " + constantsMap.size()); + constantsMap.clear(); + logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); + Instant start = Instant.now(); + logger.info("Start time: " + start); + Instant finish = Instant.now(); + long timeElapsed = Duration.between(start, finish).toMillis(); + logger.info("Time Elapsed: " + timeElapsed); + + // Send response + response.setHeader("updatedMap", ""); + dataImporterForm.getColumnPairs().clear(); + } + return null; } - return null; - } - return mapping.findForward("importData"); + return mapping.findForward("importData"); + } + return null; } + private void removeMapItem(Map map,String columnName, String selectedField) { // Check if the entry's key and value match the criteria From 5066679dbafa9c6be7f475c5e92754be141b3394 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 20 May 2024 11:44:48 +0300 Subject: [PATCH 006/229] AMP-30858- Create txt importer --- .../src/org/digijava/module/aim/action/DataImporter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index ec133383dde..88e20f5b33a 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -94,6 +94,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } workbook.close(); + } else if (Objects.equals(request.getParameter("fileType"), "text")) { CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); @@ -126,6 +127,8 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet dataImporterForm.getColumnPairs().clear(); } + return null; + } if (Objects.equals(request.getParameter("action"), "addField")) { @@ -247,8 +250,6 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } return mapping.findForward("importData"); - } - return null; } From 830dc2e7cc61fa939e76a450bc5233a132076a3e Mon Sep 17 00:00:00 2001 From: brianbrix Date: Thu, 23 May 2024 11:16:00 +0300 Subject: [PATCH 007/229] AMP-30885- Create txt importer --- .../{ => dataimporter}/DataImporter.java | 450 +---------------- .../aim/action/dataimporter/ImporterUtil.java | 476 ++++++++++++++++++ .../action/dataimporter/TxtDataImporter.java | 119 +++++ .../module/aim/util/TxtDataImporter.java | 58 +++ amp/repository/aim/struts-config.xml | 2 +- 5 files changed, 681 insertions(+), 424 deletions(-) rename amp/WEB-INF/src/org/digijava/module/aim/action/{ => dataimporter}/DataImporter.java (51%) create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java similarity index 51% rename from amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index 88e20f5b33a..83568147068 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -1,13 +1,10 @@ -package org.digijava.module.aim.action; +package org.digijava.module.aim.action.dataimporter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.CSVParser; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; +import com.opencsv.*; import com.opencsv.exceptions.CsvValidationException; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; @@ -56,11 +53,11 @@ import java.util.stream.Collectors; import static com.fasterxml.jackson.core.JsonGenerator.Feature.ESCAPE_NON_ASCII; +import static org.digijava.module.aim.action.dataimporter.ImporterUtil.*; public class DataImporter extends Action { static Logger logger = LoggerFactory.getLogger(DataImporter.class); private static final int BATCH_SIZE = 1000; - private static Map constantsMap = new HashMap<>(); @Override @@ -108,7 +105,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info("File is empty or does not contain headers."); } } catch (IOException | CsvValidationException e) { - e.printStackTrace(); + logger.error("An error occurred during extraction of headers.",e); } } @@ -221,20 +218,26 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } else { // Proceed with processing the file ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); - // Process the file in batches - int res = processFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); - if (res != 1) { - // Handle error - logger.info("Error processing file " + tempFile); - response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); + if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { + + // Process the file in batches + int res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); + if (res != 1) { + // Handle error + logger.info("Error processing file " + tempFile); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + response.setStatus(400); + return mapping.findForward("importData"); + } + } else if ( Objects.equals(request.getParameter("fileType"), "text")) { + } + // Clean up Files.delete(tempFile.toPath()); - logger.info("Cache map size: " + constantsMap.size()); - constantsMap.clear(); + logger.info("Cache map size: " + ConstantsMap.size()); + ConstantsMap.clear(); logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); Instant start = Instant.now(); logger.info("Start time: " + start); @@ -253,35 +256,8 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } - private void removeMapItem(Map map,String columnName, String selectedField) - { - // Check if the entry's key and value match the criteria - // Remove the entry - map.entrySet().removeIf(entry -> columnName.equals(entry.getKey()) && selectedField.equals(entry.getValue())); - } - - - public static boolean isFileReadable(File file) { - if (file == null || !file.exists() || !file.isFile()) { - return false; - } - return file.canRead(); - } - - // Check if the file content is valid - public static boolean isFileContentValid(File file) { - // Define your validation criteria here - // For example, let's say we want to check if the file contains at least one line - try (BufferedReader reader = new BufferedReader(new FileReader(file))) { - String line = reader.readLine(); - return line != null; // If at least one line exists, consider the content valid - } catch (IOException e) { - e.printStackTrace(); // Handle the exception appropriately - return false; // Consider the content invalid if an exception occurs - } - } - public int processFileInBatches(ImportedFilesRecord importedFilesRecord,File file, HttpServletRequest request,Map config) { + public int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) { // Open the workbook int res=0; ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); @@ -310,6 +286,7 @@ public int processFileInBatches(ImportedFilesRecord importedFilesRecord,File fil } + private void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { // Get the number of rows in the sheet int rowCount = sheet.getPhysicalNumberOfRows(); @@ -380,19 +357,19 @@ private void processBatch(List batch,Sheet sheet, HttpServletRequest reques updateOrgs(importDataModel, cell.getStringCellValue().trim(), session, "donor"); break; case "{fundingItem}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,true, "Actual"); + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell,true,true, "Actual"); break; case "{plannedCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Planned"); + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell,true,false, "Planned"); break; case "{plannedDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Planned"); + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell,false,true, "Planned"); break; case "{actualCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Actual"); + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell,true,false, "Actual"); break; case "{actualDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Actual"); + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell,false,true, "Actual"); break; default: logger.error("Unexpected value: " + entry.getValue()); @@ -408,380 +385,7 @@ private void processBatch(List batch,Sheet sheet, HttpServletRequest reques } } - private void setAFundingItem(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell,boolean commitment, boolean disbursement, String - adjustmentType) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); - String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); - String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - - if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) - { - if (!config.containsValue("{donorAgency}")) - { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); - - } - else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); - updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); - } - - }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); - } - } - - public K getKey(Map map, V value) { - for (Map.Entry entry : map.entrySet()) { - if (entry.getValue().equals(value)) { - return entry.getKey(); - } - } - return null; - } - - public static String findYearSubstring(String text) { - Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); - Matcher matcher = pattern.matcher(text); - if (matcher.find()) { - return matcher.group(); - } else { - return null; - } - } - - private String getFundingDate(String yearString) - { - LocalDate date = LocalDate.of(Integer.parseInt(yearString), 1, 1); - - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - return date.format(formatter); - } - private void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String - adjustmentType) { - String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; - Long currencyId = getCurrencyId(session); - Long adjType = getCategoryValue(session, "adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY,catHql,adjustmentType ); - Long assType = getCategoryValue(session, "assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, catHql,assistanceType); - Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); - Long orgRole = getOrganizationRole(session); - - String yearString = findYearSubstring(columnHeader); - String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); - - Funding funding = new Funding(); - funding.setDonor_organization_id(orgId); - funding.setType_of_assistance(assType); - funding.setFinancing_instrument(finInstrument); - funding.setSource_role(orgRole); - - Transaction transaction = new Transaction(); - transaction.setCurrency(currencyId); - transaction.setAdjustment_type(adjType); - transaction.setTransaction_amount(amount.doubleValue()); - transaction.setTransaction_date(fundingDate); - if (commitment) { - funding.getCommitments().add(transaction); - } - if (disbursement) { - funding.getDisbursements().add(transaction); - } - - DonorOrganization donorOrganization = new DonorOrganization(); - donorOrganization.setOrganization(orgId); - donorOrganization.setPercentage(100.0); - - importDataModel.getDonor_organization().add(donorOrganization); - importDataModel.getFundings().add(funding); - } - - - private Long getOrganizationRole(Session session) { - - if (constantsMap.containsKey("orgRole")) { - Long val= constantsMap.get("orgRole"); - logger.info("In cache... orgRole: "+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampRoleId FROM " + AmpRole.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; - - Query query = session.createQuery(hql); - query.setParameter("name", "%donor%"); - List orgRoles = query.list(); - Long orgRole = orgRoles.get(0); - constantsMap.put("orgRole", orgRole); - return orgRole; - } - - private Long getCurrencyId(Session session) { - - if (constantsMap.containsKey("currencyId")) { - Long val = constantsMap.get("currencyId"); - logger.info("In cache... currency: "+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT ac.ampCurrencyId FROM " + AmpCurrency.class.getName() + " ac " + - "WHERE ac.currencyCode = :currencyCode"; - - Query query = session.createQuery(hql); - query.setString("currencyCode", "XOF"); - Long currencyId = (Long) query.uniqueResult(); - constantsMap.put("currencyId", currencyId); - return currencyId; - } - - private Long getCategoryValue(Session session, String constantKey, String categoryKey, String hql, String possibleValue) { - String fullKey=constantKey+"_"+possibleValue; - if (constantsMap.containsKey(fullKey)) { - Long val = constantsMap.get(fullKey); - logger.info("In cache... "+fullKey+":"+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - Query query = session.createQuery(hql); - query.setParameter("categoryKey", categoryKey); - List values = query.list(); - Long categoryId = ((AmpCategoryValue) values.get(0)).getId(); - - if (!Objects.equals(possibleValue, "") && !Objects.equals(possibleValue, null)) - { - for (Object categoryValue : values) - { - if (Objects.equals(((AmpCategoryValue) categoryValue).getValue().toLowerCase(), possibleValue.toLowerCase())) - { - categoryId = ((AmpCategoryValue) categoryValue).getId(); - logger.info("Found category: "+((AmpCategoryValue) categoryValue).getValue()); - break; - } - - } - } - logger.info("Found category: "+categoryId +" for "+constantKey+"_"+possibleValue); - constantsMap.put(fullKey, categoryId); - return categoryId; - } - - private AmpActivityVersion existingActivity(ImportDataModel importDataModel,Session session) - { - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + - "WHERE a.name = :name"; - Query query= session.createQuery(hql); - query.setString("name", importDataModel.getProject_title()); - List ampActivityVersions=query.list(); - return !ampActivityVersions.isEmpty()? ampActivityVersions.get(ampActivityVersions.size()-1) :null; - } - private void setStatus(ImportDataModel importDataModel,Session session) - { - String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s " + - "JOIN s.ampCategoryClass c " + - "WHERE c.keyName = :categoryKey"; - Long statusId = getCategoryValue(session,"statusId",CategoryConstants.ACTIVITY_STATUS_KEY,hql,""); - importDataModel.setActivity_status(statusId); - - } - private void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject) throws JsonProcessingException { - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - ActivityImportRules rules = new ActivityImportRules(true, false, - true); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization - objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); - - Map map = objectMapper - .convertValue(importDataModel, new TypeReference>() {}); - JsonApiResponse response; - AmpActivityVersion existing = existingActivity(importDataModel,session); - logger.info("Existing ?"+existing); - logger.info("Data model object: "+importDataModel); - if (existing==null){ - logger.info("New activity"); - importedProject.setNewProject(true); - response= ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); - } - else - { - logger.info("Existing activity"); - importedProject.setNewProject(false); - importDataModel.setInternal_id(existing.getAmpActivityId()); - importDataModel.setAmp_id(existing.getAmpId()); - ActivityGroup activityGroup= new ActivityGroup(); - activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); - importDataModel.setActivity_group(activityGroup); - map = objectMapper - .convertValue(importDataModel, new TypeReference>() {}); - response= ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); - - } - if (response!=null) { - if (!response.getErrors().isEmpty()) { - importedProject.setImportStatus(ImportStatus.FAILED); - } else { - importedProject.setImportStatus(ImportStatus.SUCCESS); - - } - } - - String resp = objectMapper.writeValueAsString(response); - importedProject.setImportResponse(resp); - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - session.saveOrUpdate(importedProject); - logger.info("Imported project: "+importedProject); - } - - - - private void updateSectors(ImportDataModel importDataModel, String name, Session session, boolean primary) - { - - if (constantsMap.containsKey("sector_"+name)) { - Long sectorId = constantsMap.get("sector_"+name); - logger.info("In cache... sector "+"sector_"+name+":"+sectorId); - createSector(importDataModel,primary,sectorId); - } - else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - - session.doWork(connection -> { - String query = primary ? "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Primary'" : "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Secondary'"; - try (PreparedStatement statement = connection.prepareStatement(query)) { - // Set the name as a parameter to the prepared statement - statement.setString(1, name); - - // Execute the query and process the results - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long ampSectorId = resultSet.getLong("amp_sector_id"); - createSector(importDataModel, primary, ampSectorId); - constantsMap.put("sector_"+name, ampSectorId); - } - } - - } catch (SQLException e) { - logger.error("Error getting sectors", e); - } - }); - } - - - - } - - private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { - Sector sector1 = new Sector(); - sector1.setSector_percentage(100.00); - sector1.setSector(ampSectorId); - if (primary) { - importDataModel.getPrimary_sectors().add(sector1); - } - else - { - importDataModel.getSecondary_sectors().add(sector1); - - } - } - - private Long getRandomOrg(Session session) - { - Long randomOrg; - if (constantsMap.containsKey("randomOrg")) { - randomOrg = constantsMap.get("randomOrg"); - logger.info("In cache... randomOrg "+randomOrg); - }else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; - - randomOrg = (Long) session.createQuery(hql).setMaxResults(1).uniqueResult(); - constantsMap.put("randomOrg",randomOrg); - } - return randomOrg; - - - } - - private void updateOrgs(ImportDataModel importDataModel, String name, Session session, String type) - { - Long orgId; - - if (constantsMap.containsKey("org_"+name)) { - orgId = constantsMap.get("org_"+name); - logger.info("In cache... organisation "+"org_"+name+":"+orgId); - } - else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; - - Query query = session.createQuery(hql); - query.setParameter("name", "%" + name + "%"); - List organisations = query.list(); - if (!organisations.isEmpty()) { - orgId = organisations.get(0); - } else { - hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; - - query = session.createQuery(hql).setMaxResults(1); - orgId = (Long) query.uniqueResult(); - } - constantsMap.put("org_" + name, orgId); - } - logger.info("Organisation: " + orgId); - - if (Objects.equals(type, "donor")) { - DonorOrganization donorOrganization = new DonorOrganization(); - donorOrganization.setOrganization(orgId); - donorOrganization.setPercentage(100.0); - importDataModel.getDonor_organization().add(donorOrganization); - } - - - - } - - - private static int getColumnIndexByName(Sheet sheet, String columnName) { - try { - Row headerRow = sheet.getRow(0); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - if (cell != null && columnName.equals(cell.getStringCellValue())) { - return i; - } - } - return -1; - }catch (Exception e) - { - logger.error("Error getting column index for "+columnName,e); - return -1; - } - - } private List getEntityFieldsInfo() { List fieldsInfos = new ArrayList<>(); fieldsInfos.add("{projectTitle}"); diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java new file mode 100644 index 00000000000..56ba7b06778 --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java @@ -0,0 +1,476 @@ +package org.digijava.module.aim.action.dataimporter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; +import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; +import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; +import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; +import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.module.admin.dbentity.ImportStatus; +import org.digijava.module.admin.dbentity.ImportedProject; +import org.digijava.module.admin.util.model.*; +import org.digijava.module.aim.dbentity.AmpActivityVersion; +import org.digijava.module.aim.dbentity.AmpCurrency; +import org.digijava.module.aim.dbentity.AmpOrganisation; +import org.digijava.module.aim.dbentity.AmpRole; +import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; +import org.digijava.module.categorymanager.util.CategoryConstants; +import org.hibernate.Query; +import org.hibernate.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.fasterxml.jackson.core.JsonGenerator.Feature.ESCAPE_NON_ASCII; + +public class ImporterUtil { + protected static Map ConstantsMap = new HashMap<>(); + + private static final int BATCH_SIZE = 1000; + private static final Logger logger = LoggerFactory.getLogger(ImporterUtil.class); + static void setAFundingItemForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, String + adjustmentType) { + int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); + String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); + String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + + if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) + { + if (!config.containsValue("{donorAgency}")) + { + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + + } + else { + int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); + updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + } + + }else { + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + } + } + + static void setAFundingItemForTxt(Map row ,Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, boolean commitment, boolean disbursement, String + adjustmentType) { +// int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); + String finInstrument= row.get(getKey(config, "{financingInstrument}")); + finInstrument = finInstrument!= null? finInstrument : ""; + + // detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); + String typeOfAss =row.get(getKey(config, "{typeOfAssistance}")); ; + typeOfAss=typeOfAss!=null? typeOfAss:""; + if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) + { + if (!config.containsValue("{donorAgency}")) + { + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + + } + else { + int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); + updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + } + + }else { + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + } + } + + public static K getKey(Map map, V value) { + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue().equals(value)) { + return entry.getKey(); + } + } + return null; + } + + public static String findYearSubstring(String text) { + Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); + Matcher matcher = pattern.matcher(text); + if (matcher.find()) { + return matcher.group(); + } else { + return null; + } + } + + protected static void removeMapItem(Map map,String columnName, String selectedField) + { + // Check if the entry's key and value match the criteria + // Remove the entry + map.entrySet().removeIf(entry -> columnName.equals(entry.getKey()) && selectedField.equals(entry.getValue())); + } + + + public static boolean isFileReadable(File file) { + if (file == null || !file.exists() || !file.isFile()) { + return false; + } + return file.canRead(); + } + + // Check if the file content is valid + public static boolean isFileContentValid(File file) { + // Define your validation criteria here + // For example, let's say we want to check if the file contains at least one line + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line = reader.readLine(); + return line != null; // If at least one line exists, consider the content valid + } catch (IOException e) { + e.printStackTrace(); // Handle the exception appropriately + return false; // Consider the content invalid if an exception occurs + } + } + + + private static String getFundingDate(String yearString) + { + LocalDate date = LocalDate.of(Integer.parseInt(yearString), 1, 1); + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + return date.format(formatter); + } + private static void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String + adjustmentType) { + String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; + Long currencyId = getCurrencyId(session); + Long adjType = getCategoryValue(session, "adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY,catHql,adjustmentType ); + Long assType = getCategoryValue(session, "assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, catHql,assistanceType); + Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); + Long orgRole = getOrganizationRole(session); + + String yearString = findYearSubstring(columnHeader); + String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + + Funding funding = new Funding(); + funding.setDonor_organization_id(orgId); + funding.setType_of_assistance(assType); + funding.setFinancing_instrument(finInstrument); + funding.setSource_role(orgRole); + + Transaction transaction = new Transaction(); + transaction.setCurrency(currencyId); + transaction.setAdjustment_type(adjType); + transaction.setTransaction_amount(amount.doubleValue()); + transaction.setTransaction_date(fundingDate); + if (commitment) { + funding.getCommitments().add(transaction); + } + if (disbursement) { + funding.getDisbursements().add(transaction); + } + + DonorOrganization donorOrganization = new DonorOrganization(); + donorOrganization.setOrganization(orgId); + donorOrganization.setPercentage(100.0); + + importDataModel.getDonor_organization().add(donorOrganization); + importDataModel.getFundings().add(funding); + } + + + private static Long getOrganizationRole(Session session) { + + if (ConstantsMap.containsKey("orgRole")) { + Long val= ConstantsMap.get("orgRole"); + logger.info("In cache... orgRole: "+val); + return val; + + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT o.ampRoleId FROM " + AmpRole.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; + + Query query = session.createQuery(hql); + query.setParameter("name", "%donor%"); + List orgRoles = query.list(); + Long orgRole = orgRoles.get(0); + ConstantsMap.put("orgRole", orgRole); + return orgRole; + } + + private static Long getCurrencyId(Session session) { + + if (ConstantsMap.containsKey("currencyId")) { + Long val = ConstantsMap.get("currencyId"); + logger.info("In cache... currency: "+val); + return val; + + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT ac.ampCurrencyId FROM " + AmpCurrency.class.getName() + " ac " + + "WHERE ac.currencyCode = :currencyCode"; + + Query query = session.createQuery(hql); + query.setString("currencyCode", "XOF"); + Long currencyId = (Long) query.uniqueResult(); + ConstantsMap.put("currencyId", currencyId); + return currencyId; + } + + private static Long getCategoryValue(Session session, String constantKey, String categoryKey, String hql, String possibleValue) { + String fullKey=constantKey+"_"+possibleValue; + if (ConstantsMap.containsKey(fullKey)) { + Long val = ConstantsMap.get(fullKey); + logger.info("In cache... "+fullKey+":"+val); + return val; + + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + Query query = session.createQuery(hql); + query.setParameter("categoryKey", categoryKey); + List values = query.list(); + Long categoryId = ((AmpCategoryValue) values.get(0)).getId(); + + if (!Objects.equals(possibleValue, "") && !Objects.equals(possibleValue, null)) + { + for (Object categoryValue : values) + { + if (Objects.equals(((AmpCategoryValue) categoryValue).getValue().toLowerCase(), possibleValue.toLowerCase())) + { + categoryId = ((AmpCategoryValue) categoryValue).getId(); + logger.info("Found category: "+((AmpCategoryValue) categoryValue).getValue()); + break; + } + + } + } + logger.info("Found category: "+categoryId +" for "+constantKey+"_"+possibleValue); + ConstantsMap.put(fullKey, categoryId); + return categoryId; + } + + private static AmpActivityVersion existingActivity(ImportDataModel importDataModel, Session session) + { + if (!session.isOpen()) { + session=PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + + "WHERE a.name = :name"; + Query query= session.createQuery(hql); + query.setString("name", importDataModel.getProject_title()); + List ampActivityVersions=query.list(); + return !ampActivityVersions.isEmpty()? ampActivityVersions.get(ampActivityVersions.size()-1) :null; + } + protected static void setStatus(ImportDataModel importDataModel,Session session) + { + String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s " + + "JOIN s.ampCategoryClass c " + + "WHERE c.keyName = :categoryKey"; + Long statusId = getCategoryValue(session,"statusId",CategoryConstants.ACTIVITY_STATUS_KEY,hql,""); + importDataModel.setActivity_status(statusId); + + } + static void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject) throws JsonProcessingException { + if (!session.isOpen()) { + session=PersistenceManager.getRequestDBSession(); + } + ActivityImportRules rules = new ActivityImportRules(true, false, + true); + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization + objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); + + Map map = objectMapper + .convertValue(importDataModel, new TypeReference>() {}); + JsonApiResponse response; + AmpActivityVersion existing = existingActivity(importDataModel,session); + logger.info("Existing ?"+existing); + logger.info("Data model object: "+importDataModel); + if (existing==null){ + logger.info("New activity"); + importedProject.setNewProject(true); + response= ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); + } + else + { + logger.info("Existing activity"); + importedProject.setNewProject(false); + importDataModel.setInternal_id(existing.getAmpActivityId()); + importDataModel.setAmp_id(existing.getAmpId()); + ActivityGroup activityGroup= new ActivityGroup(); + activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); + importDataModel.setActivity_group(activityGroup); + map = objectMapper + .convertValue(importDataModel, new TypeReference>() {}); + response= ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); + + } + if (response!=null) { + if (!response.getErrors().isEmpty()) { + importedProject.setImportStatus(ImportStatus.FAILED); + } else { + importedProject.setImportStatus(ImportStatus.SUCCESS); + + } + } + + String resp = objectMapper.writeValueAsString(response); + importedProject.setImportResponse(resp); + if (!session.isOpen()) { + session=PersistenceManager.getRequestDBSession(); + } + session.saveOrUpdate(importedProject); + logger.info("Imported project: "+importedProject); + } + + + + static void updateSectors(ImportDataModel importDataModel, String name, Session session, boolean primary) + { + + if (ConstantsMap.containsKey("sector_"+name)) { + Long sectorId = ConstantsMap.get("sector_"+name); + logger.info("In cache... sector "+"sector_"+name+":"+sectorId); + createSector(importDataModel,primary,sectorId); + } + else { + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + + session.doWork(connection -> { + String query = primary ? "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Primary'" : "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Secondary'"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + // Set the name as a parameter to the prepared statement + statement.setString(1, name); + + // Execute the query and process the results + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + Long ampSectorId = resultSet.getLong("amp_sector_id"); + createSector(importDataModel, primary, ampSectorId); + ConstantsMap.put("sector_"+name, ampSectorId); + } + } + + } catch (SQLException e) { + logger.error("Error getting sectors", e); + } + }); + } + + + + } + + private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { + Sector sector1 = new Sector(); + sector1.setSector_percentage(100.00); + sector1.setSector(ampSectorId); + if (primary) { + importDataModel.getPrimary_sectors().add(sector1); + } + else + { + importDataModel.getSecondary_sectors().add(sector1); + + } + } + + private static Long getRandomOrg(Session session) + { + Long randomOrg; + if (ConstantsMap.containsKey("randomOrg")) { + randomOrg = ConstantsMap.get("randomOrg"); + logger.info("In cache... randomOrg "+randomOrg); + }else { + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; + + randomOrg = (Long) session.createQuery(hql).setMaxResults(1).uniqueResult(); + ConstantsMap.put("randomOrg",randomOrg); + } + return randomOrg; + + + } + + static void updateOrgs(ImportDataModel importDataModel, String name, Session session, String type) + { + Long orgId; + + if (ConstantsMap.containsKey("org_"+name)) { + orgId = ConstantsMap.get("org_"+name); + logger.info("In cache... organisation "+"org_"+name+":"+orgId); + } + else { + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; + + Query query = session.createQuery(hql); + query.setParameter("name", "%" + name + "%"); + List organisations = query.list(); + if (!organisations.isEmpty()) { + orgId = organisations.get(0); + } else { + hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; + + query = session.createQuery(hql).setMaxResults(1); + orgId = (Long) query.uniqueResult(); + } + ConstantsMap.put("org_" + name, orgId); + } + logger.info("Organisation: " + orgId); + + if (Objects.equals(type, "donor")) { + DonorOrganization donorOrganization = new DonorOrganization(); + donorOrganization.setOrganization(orgId); + donorOrganization.setPercentage(100.0); + importDataModel.getDonor_organization().add(donorOrganization); + } + + + + } + protected static int getColumnIndexByName(Sheet sheet, String columnName) { + try { + Row headerRow = sheet.getRow(0); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + if (cell != null && columnName.equals(cell.getStringCellValue())) { + return i; + } + } + return -1; + }catch (Exception e) + { + logger.error("Error getting column index for "+columnName,e); + return -1; + } + + } +} diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java new file mode 100644 index 00000000000..17fd5a686d6 --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -0,0 +1,119 @@ +package org.digijava.module.aim.action.dataimporter; + +import com.opencsv.CSVReaderHeaderAware; +import com.opencsv.exceptions.CsvValidationException; +import org.dgfoundation.amp.onepager.util.SessionUtil; +import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.module.admin.dbentity.ImportedFilesRecord; +import org.digijava.module.admin.dbentity.ImportedProject; +import org.digijava.module.admin.util.model.ImportDataModel; +import org.digijava.module.aim.util.TeamMemberUtil; +import org.hibernate.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.digijava.module.aim.action.dataimporter.ImporterUtil.*; + +public class TxtDataImporter { + private static final int BATCH_SIZE = 1000; + private static final Logger logger = LoggerFactory.getLogger(TxtDataImporter.class); + + + private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) + { + try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { + List> batch = new ArrayList<>(); + Map values; + + while ((values = reader.readMap()) != null) { + batch.add(values); + + if (batch.size() == BATCH_SIZE) { + // Process the batch + processBatch(batch); + // Clear the batch for the next set of rows + batch.clear(); + } + } + + // Process any remaining rows in the batch + if (!batch.isEmpty()) { + processBatch(batch); + } + } catch (IOException | CsvValidationException e) { + e.printStackTrace(); + } + } + + + private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) { + SessionUtil.extendSessionIfNeeded(request); + Session session = PersistenceManager.getRequestDBSession(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + for (Map row : batch) { + ImportedProject importedProject= new ImportedProject(); + importedProject.setImportedFilesRecord(importedFilesRecord); + + ImportDataModel importDataModel = new ImportDataModel(); + importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); + importDataModel.setIs_draft(true); + OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); + importDataModel.setCreation_date(now.format(formatter)); + setStatus(importDataModel, session); + for (Map.Entry entry : config.entrySet()) { + switch (entry.getValue()) { + case "{projectTitle}": + importDataModel.setProject_title(row.get(entry.getKey())); + break; + case "{projectDescription}": + importDataModel.setDescription(row.get(entry.getKey().trim())); + break; + case "{projectLocation}": +// ampActivityVersion.addLocation(new AmpActivityLocation()); + break; + case "{primarySector}": + updateSectors(importDataModel, row.get(entry.getKey().trim()), session, true); + break; + case "{secondarySector}": + updateSectors(importDataModel, row.get(entry.getKey().trim()), session, false); + break; + case "{donorAgency}": + updateOrgs(importDataModel,row.get(entry.getKey().trim()), session, "donor"); + break; + case "{fundingItem}": + setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,true, "Actual"); + break; + case "{plannedCommitment}": + setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Planned"); + break; + case "{plannedDisbursement}": + setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Planned"); + break; + case "{actualCommitment}": + setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Actual"); + break; + case "{actualDisbursement}": + setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Actual"); + break; + default: + logger.error("Unexpected value: " + entry.getValue()); + break; + } + } + } + } +} diff --git a/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java new file mode 100644 index 00000000000..19514f4564b --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java @@ -0,0 +1,58 @@ +package org.digijava.module.aim.util; + +import com.opencsv.CSVReaderHeaderAware; +import com.opencsv.exceptions.CsvValidationException; +import org.digijava.module.admin.dbentity.ImportedFilesRecord; + +import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TxtDataImporter { + private static final int BATCH_SIZE = 1000; + + private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) + { + try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { + List> batch = new ArrayList<>(); + Map values; + + while ((values = reader.readMap()) != null) { + batch.add(values); + + if (batch.size() == BATCH_SIZE) { + // Process the batch + processBatch(batch); + // Clear the batch for the next set of rows + batch.clear(); + } + } + + // Process any remaining rows in the batch + if (!batch.isEmpty()) { + processBatch(batch); + } + } catch (IOException | CsvValidationException e) { + e.printStackTrace(); + } + } + + + private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) { + for (Map row : batch) { + // Access data by column names (headers) + String header1Value = row.get("Header1"); + String header2Value = row.get("Header2"); + String header3Value = row.get("Header3"); + + // Do something with the extracted data + System.out.println("Header1: " + header1Value); + System.out.println("Header2: " + header2Value); + System.out.println("Header3: " + header3Value); + } + } +} diff --git a/amp/repository/aim/struts-config.xml b/amp/repository/aim/struts-config.xml index fe3b300533f..801657f82a9 100644 --- a/amp/repository/aim/struts-config.xml +++ b/amp/repository/aim/struts-config.xml @@ -1355,7 +1355,7 @@ - + From 5bae9ee3621047eade968e0d87d75e5fd3fef9ea Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 13:04:50 +0300 Subject: [PATCH 008/229] AMP-30858- Create txt importer for activities --- .../aim/action/dataimporter/ImporterUtil.java | 14 +++++++------- .../aim/action/dataimporter/TxtDataImporter.java | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java index 56ba7b06778..69cdc146448 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java @@ -71,30 +71,30 @@ static void setAFundingItemForExcel(Sheet sheet, Map config, Row } } - static void setAFundingItemForTxt(Map row ,Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, boolean commitment, boolean disbursement, String + static void setAFundingItemForTxt(Map row ,Map config, Map.Entry entry, ImportDataModel importDataModel, Session session,Number value, boolean commitment, boolean disbursement, String adjustmentType) { // int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); String finInstrument= row.get(getKey(config, "{financingInstrument}")); finInstrument = finInstrument!= null? finInstrument : ""; // detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); - String typeOfAss =row.get(getKey(config, "{typeOfAssistance}")); ; + String typeOfAss =row.get(getKey(config, "{typeOfAssistance}")); typeOfAss=typeOfAss!=null? typeOfAss:""; if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) { if (!config.containsValue("{donorAgency}")) { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, value, entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); } else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); - updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + String donorColumn = row.get(getKey(config, "{donorAgency}")); + updateOrgs(importDataModel, donorColumn!=null && !donorColumn.isEmpty() ? donorColumn.trim():"no org", session, "donor"); + updateFunding(importDataModel, session, value, entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); } }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, value, entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); } } diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 17fd5a686d6..2838f56622a 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -41,7 +41,7 @@ private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, Fi if (batch.size() == BATCH_SIZE) { // Process the batch - processBatch(batch); + processBatch(batch, request,config,importedFilesRecord); // Clear the batch for the next set of rows batch.clear(); } @@ -49,7 +49,7 @@ private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, Fi // Process any remaining rows in the batch if (!batch.isEmpty()) { - processBatch(batch); + processBatch(batch, request,config,importedFilesRecord); } } catch (IOException | CsvValidationException e) { e.printStackTrace(); @@ -95,19 +95,19 @@ private static void processBatch(List> batch, HttpServletRe updateOrgs(importDataModel,row.get(entry.getKey().trim()), session, "donor"); break; case "{fundingItem}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,true, "Actual"); + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())),true,true, "Actual"); break; case "{plannedCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Planned"); + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())),true,false, "Planned"); break; case "{plannedDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Planned"); + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())),false,true, "Planned"); break; case "{actualCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Actual"); + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())),true,false, "Actual"); break; case "{actualDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Actual"); + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())),false,true, "Actual"); break; default: logger.error("Unexpected value: " + entry.getValue()); From 5ad1d3c8a7e949b59d48b769902a697a5d590d71 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 14:15:04 +0300 Subject: [PATCH 009/229] AMP-30858- Create txt importer for activities --- .../aim/action/dataimporter/DataImporter.java | 30 +++++++------------ .../action/dataimporter/TxtDataImporter.java | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index 83568147068..70b716baef8 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -1,10 +1,11 @@ package org.digijava.module.aim.action.dataimporter; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.*; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvValidationException; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; @@ -18,22 +19,14 @@ import org.apache.struts.action.ActionForward; import org.apache.struts.action.ActionMapping; import org.dgfoundation.amp.onepager.util.SessionUtil; -import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; -import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; -import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; -import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; import org.digijava.kernel.persistence.PersistenceManager; import org.digijava.module.admin.dbentity.ImportStatus; import org.digijava.module.admin.dbentity.ImportedFilesRecord; import org.digijava.module.admin.dbentity.ImportedProject; import org.digijava.module.admin.util.ImportedFileUtil; -import org.digijava.module.admin.util.model.*; -import org.digijava.module.aim.dbentity.*; +import org.digijava.module.admin.util.model.ImportDataModel; import org.digijava.module.aim.form.DataImporterForm; import org.digijava.module.aim.util.TeamMemberUtil; -import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; -import org.digijava.module.categorymanager.util.CategoryConstants; -import org.hibernate.Query; import org.hibernate.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,17 +35,14 @@ import javax.servlet.http.HttpServletResponse; import java.io.*; import java.nio.file.Files; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.fasterxml.jackson.core.JsonGenerator.Feature.ESCAPE_NON_ASCII; import static org.digijava.module.aim.action.dataimporter.ImporterUtil.*; public class DataImporter extends Action { @@ -230,7 +220,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet return mapping.findForward("importData"); } } else if ( Objects.equals(request.getParameter("fileType"), "text")) { - + TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); } diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 2838f56622a..649ebe209ac 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -30,7 +30,7 @@ public class TxtDataImporter { private static final Logger logger = LoggerFactory.getLogger(TxtDataImporter.class); - private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) + public static void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) { try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { List> batch = new ArrayList<>(); From 293becda4cf9e3d38cfb08699f607d529f7976a7 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 14:19:46 +0300 Subject: [PATCH 010/229] AMP-30858- Create txt importer for activities --- .../module/aim/util/TxtDataImporter.java | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java diff --git a/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java deleted file mode 100644 index 19514f4564b..00000000000 --- a/amp/WEB-INF/src/org/digijava/module/aim/util/TxtDataImporter.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.digijava.module.aim.util; - -import com.opencsv.CSVReaderHeaderAware; -import com.opencsv.exceptions.CsvValidationException; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; - -import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class TxtDataImporter { - private static final int BATCH_SIZE = 1000; - - private void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) - { - try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { - List> batch = new ArrayList<>(); - Map values; - - while ((values = reader.readMap()) != null) { - batch.add(values); - - if (batch.size() == BATCH_SIZE) { - // Process the batch - processBatch(batch); - // Clear the batch for the next set of rows - batch.clear(); - } - } - - // Process any remaining rows in the batch - if (!batch.isEmpty()) { - processBatch(batch); - } - } catch (IOException | CsvValidationException e) { - e.printStackTrace(); - } - } - - - private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) { - for (Map row : batch) { - // Access data by column names (headers) - String header1Value = row.get("Header1"); - String header2Value = row.get("Header2"); - String header3Value = row.get("Header3"); - - // Do something with the extracted data - System.out.println("Header1: " + header1Value); - System.out.println("Header2: " + header2Value); - System.out.println("Header3: " + header3Value); - } - } -} From e09fa51a3a9aa238327e434ee3d6e3b97608569b Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 14:30:52 +0300 Subject: [PATCH 011/229] AMP-30858- Create txt importer for activities --- amp/repository/aim/view/dataImporter.jsp | 2 -- 1 file changed, 2 deletions(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index 2783d9c680c..4a13403003e 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -81,10 +81,8 @@ $('#template-file').attr("accept", ".json"); $('#separator-div').hide(); } - }) }); - }); function sendValuesToBackend(columnName, selectedField, action) { // Create a FormData object to send data in the request body From 4fc53eb475e4b428e3ff26bdd8ff3b65f37d97f1 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 14:38:56 +0300 Subject: [PATCH 012/229] AMP-30858- Create txt importer for activities --- amp/repository/aim/view/dataImporter.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index 4a13403003e..f7ca1b87a07 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -54,7 +54,7 @@ }); }); }); - $('#file-type').change(function() { + $('.file_type').change(function() { var fileType = $(this).val(); console.log("File type selected: " + fileType==='excel'); if (fileType === "csv") { @@ -248,7 +248,7 @@
- From d0b0851fbc4c890da641a63cb0790b997a66d696 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 14:47:14 +0300 Subject: [PATCH 013/229] AMP-30858- Create txt importer for activities --- amp/repository/aim/view/dataImporter.jsp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index f7ca1b87a07..fe0bf6dc2fe 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -53,7 +53,7 @@ $(this).remove(); }); }); - }); + $('.file_type').change(function() { var fileType = $(this).val(); console.log("File type selected: " + fileType==='excel'); @@ -82,6 +82,7 @@ $('#separator-div').hide(); } }); + }); function sendValuesToBackend(columnName, selectedField, action) { From d0556b457101dde01d52ec6dfed3138c17574dcd Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 16:02:23 +0300 Subject: [PATCH 014/229] AMP-30858- Create txt importer for activities --- amp/repository/aim/view/dataImporter.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index fe0bf6dc2fe..cfaa81a9fc1 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -163,7 +163,7 @@ function uploadTemplateFile() { var formData = new FormData(); - var fileInput = document.getElementById('templateFile'); + var fileInput = document.getElementById('template-file'); var fileType = $('#file-type').val(); var dataSeparator = $('#data-separator').val(); @@ -195,7 +195,7 @@ function uploadDataFile() { var formData = new FormData(); - var fileInput = document.getElementById('dataFile'); + var fileInput = document.getElementById('data-file'); // Check if a file is selected if (!fileInput.files.length) { alert("Please select a file to upload."); From 0f3a895a47e097fd6f954a08a39aaf3837de5dd5 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 28 May 2024 16:30:52 +0300 Subject: [PATCH 015/229] AMP-30858- Create txt importer for activities --- .../aim/action/dataimporter/DataImporter.java | 23 ++++++++++--------- .../action/dataimporter/TxtDataImporter.java | 6 +++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index 70b716baef8..e009491c100 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -162,7 +162,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { - logger.info("This is the action Upload " + request.getParameter("uploadDataFile")); + logger.info("This is the action Upload " + request.getParameter("action")); String fileName = dataImporterForm.getDataFile().getFileName(); String tempDirPath = System.getProperty("java.io.tmpdir"); File tempDir = new File(tempDirPath); @@ -207,24 +207,26 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet // Optionally, you can respond with an error message to the client. } else { // Proceed with processing the file + int res = 0; ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { // Process the file in batches - int res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); - if (res != 1) { - // Handle error - logger.info("Error processing file " + tempFile); - response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); - } + res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); } else if ( Objects.equals(request.getParameter("fileType"), "text")) { - TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); + res=TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); + } + if (res != 1) { + // Handle error + logger.info("Error processing file " + tempFile); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + response.setStatus(400); + return mapping.findForward("importData"); } // Clean up + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.SUCCESS); Files.delete(tempFile.toPath()); logger.info("Cache map size: " + ConstantsMap.size()); ConstantsMap.clear(); @@ -263,7 +265,6 @@ public int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, Fi } logger.info("Closing the workbook..."); - ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.SUCCESS); res =1; } catch (IOException e) { ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 649ebe209ac..200e370e40a 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -30,8 +30,9 @@ public class TxtDataImporter { private static final Logger logger = LoggerFactory.getLogger(TxtDataImporter.class); - public static void processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) + public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) { + logger.info("Processing txt file: " + file.getName()); try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { List> batch = new ArrayList<>(); Map values; @@ -52,8 +53,9 @@ public static void processTxtFileInBatches(ImportedFilesRecord importedFilesReco processBatch(batch, request,config,importedFilesRecord); } } catch (IOException | CsvValidationException e) { - e.printStackTrace(); + return 0; } + return 1; } From ffabe626c6505b0904da80ad966ec0128c6d5ff0 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 29 May 2024 12:44:22 +0300 Subject: [PATCH 016/229] AMP-30858- Create txt importer for activities --- .../digijava/module/aim/action/dataimporter/TxtDataImporter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 200e370e40a..cf09d3f0451 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -53,6 +53,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor processBatch(batch, request,config,importedFilesRecord); } } catch (IOException | CsvValidationException e) { + logger.error("Error processing txt file "+e.getMessage(),e); return 0; } return 1; From e945d84f6013972cfe595a68eceb08ad2d84e18d Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 29 May 2024 14:02:37 +0300 Subject: [PATCH 017/229] AMP-30858- Create txt importer for activities --- .../module/aim/action/dataimporter/DataImporter.java | 2 +- .../module/aim/action/dataimporter/TxtDataImporter.java | 9 ++++++++- amp/repository/aim/view/dataImporter.jsp | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index e009491c100..457a2d5cf02 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -162,7 +162,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { - logger.info("This is the action Upload " + request.getParameter("action")); + logger.info("This is the action " + request.getParameter("action")); String fileName = dataImporterForm.getDataFile().getFileName(); String tempDirPath = System.getProperty("java.io.tmpdir"); File tempDir = new File(tempDirPath); diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index cf09d3f0451..cff429af653 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -1,6 +1,9 @@ package org.digijava.module.aim.action.dataimporter; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReaderHeaderAware; +import com.opencsv.CSVReaderHeaderAwareBuilder; import com.opencsv.exceptions.CsvValidationException; import org.dgfoundation.amp.onepager.util.SessionUtil; import org.digijava.kernel.persistence.PersistenceManager; @@ -33,12 +36,16 @@ public class TxtDataImporter { public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config) { logger.info("Processing txt file: " + file.getName()); - try (CSVReaderHeaderAware reader = new CSVReaderHeaderAware(new FileReader(file))) { + CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); + +// try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { + try (CSVReaderHeaderAware reader = new CSVReaderHeaderAwareBuilder(new FileReader(file)).withCSVParser(parser).withSkipLines(1).build()) { List> batch = new ArrayList<>(); Map values; while ((values = reader.readMap()) != null) { batch.add(values); + logger.info("Batch this far: "+batch); if (batch.size() == BATCH_SIZE) { // Process the batch diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index cfaa81a9fc1..e1d554e16f6 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -195,6 +195,8 @@ function uploadDataFile() { var formData = new FormData(); + var fileType = $('#file-type').val(); + var dataSeparator = $('#data-separator').val(); var fileInput = document.getElementById('data-file'); // Check if a file is selected if (!fileInput.files.length) { @@ -203,6 +205,8 @@ } formData.append('dataFile', fileInput.files[0]); formData.append('action',"uploadDataFile"); + formData.append('fileType', fileType); + formData.append('dataSeparator', dataSeparator); var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); From 2d6060c55624f1f84400153f482820460ed6c389 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 29 May 2024 14:17:35 +0300 Subject: [PATCH 018/229] AMP-30858- Create txt importer for activities --- .../module/aim/action/dataimporter/TxtDataImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index cff429af653..055ef1177dc 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -39,7 +39,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); // try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { - try (CSVReaderHeaderAware reader = new CSVReaderHeaderAwareBuilder(new FileReader(file)).withCSVParser(parser).withSkipLines(1).build()) { + try (CSVReaderHeaderAware reader = new CSVReaderHeaderAwareBuilder(new FileReader(file)).withCSVParser(parser).build()) { List> batch = new ArrayList<>(); Map values; From 30c0db84fedc1311b7bd7429479a4cba10c2681f Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 29 May 2024 14:33:36 +0300 Subject: [PATCH 019/229] AMP-30858- Create txt importer for activities --- .../module/aim/action/dataimporter/TxtDataImporter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 055ef1177dc..0670d79219d 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -68,6 +68,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) { + logger.info("Processing txt batch"); SessionUtil.extendSessionIfNeeded(request); Session session = PersistenceManager.getRequestDBSession(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); @@ -84,6 +85,7 @@ private static void processBatch(List> batch, HttpServletRe OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); setStatus(importDataModel, session); + logger.info("Configuration: "+config); for (Map.Entry entry : config.entrySet()) { switch (entry.getValue()) { case "{projectTitle}": From 7e43f163d04cbbd2c793aba56e0c46dee3ec0d19 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 29 May 2024 14:43:55 +0300 Subject: [PATCH 020/229] AMP-30858- Create txt importer for activities --- .../module/aim/action/dataimporter/TxtDataImporter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 0670d79219d..d6b7932cb03 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -1,5 +1,6 @@ package org.digijava.module.aim.action.dataimporter; +import com.fasterxml.jackson.core.JsonProcessingException; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReaderHeaderAware; @@ -67,7 +68,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor } - private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) { + private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { logger.info("Processing txt batch"); SessionUtil.extendSessionIfNeeded(request); Session session = PersistenceManager.getRequestDBSession(); @@ -126,6 +127,9 @@ private static void processBatch(List> batch, HttpServletRe break; } } + importTheData(importDataModel, session, importedProject); + } + } } From 934920093a6559a028872157e1d66985394c6dfc Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 5 Jun 2024 22:26:17 +0300 Subject: [PATCH 021/229] AMP-30885 Mapping Ecowas file with importer --- .../module/aim/action/DataImporter.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index c66003a7ef5..367ce0f0016 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -340,6 +340,7 @@ private void processBatch(List batch,Sheet sheet, HttpServletRequest reques case "{projectDescription}": importDataModel.setDescription(cell.getStringCellValue().trim()); break; + case "{projectStatus}": case "{projectLocation}": // ampActivityVersion.addLocation(new AmpActivityLocation()); break; @@ -381,28 +382,31 @@ private void processBatch(List batch,Sheet sheet, HttpServletRequest reques } } + + private void setAFundingItem(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell,boolean commitment, boolean disbursement, String adjustmentType) { int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + String separateFundingDate=!config.containsValue("{transactionDate}")?null:getKey(config, "{transactionDate}"); if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) { if (!config.containsValue("{donorAgency}")) { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate, getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); } else { int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); } }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate,new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); } } @@ -434,7 +438,7 @@ private String getFundingDate(String yearString) return date.format(formatter); } - private void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String + private void updateFunding(ImportDataModel importDataModel, Session session, Number amount,String separateFundingDate, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String adjustmentType) { String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; Long currencyId = getCurrencyId(session); @@ -443,7 +447,15 @@ private void updateFunding(ImportDataModel importDataModel, Session session, Num Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); Long orgRole = getOrganizationRole(session); - String yearString = findYearSubstring(columnHeader); + String yearString; + if (separateFundingDate!=null) + { + yearString=findYearSubstring(separateFundingDate); + } + else { + yearString=findYearSubstring(columnHeader); + } + String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); Funding funding = new Funding(); @@ -772,6 +784,7 @@ private List getEntityFieldsInfo() { fieldsInfos.add("{plannedDisbursement}"); fieldsInfos.add("{plannedCommitment}"); fieldsInfos.add("{fundingItem}"); + fieldsInfos.add("{transactionDate}"); fieldsInfos.add("{financingInstrument}"); fieldsInfos.add("{typeOfAssistance}"); fieldsInfos.add("{secondarySubSector}"); From 5a5f43d08e6a0a66a279dc9e6e1d0aedf0683eaf Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 5 Jun 2024 22:47:31 +0300 Subject: [PATCH 022/229] AMP-30885 Mapping Ecowas file with importer --- .../module/aim/action/DataImporter.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index 367ce0f0016..4d22a37c41d 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -390,23 +390,26 @@ private void setAFundingItem(Sheet sheet, Map config, Row row, M String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - String separateFundingDate=!config.containsValue("{transactionDate}")?null:getKey(config, "{transactionDate}"); + int separateFundingDateColumn=getColumnIndexByName(sheet, getKey(config, "{transactionDate}")); + String separateFundingDate = separateFundingDateColumn>=0? row.getCell(separateFundingDateColumn).getStringCellValue(): null; + int currencyCodeColumn=getColumnIndexByName(sheet, getKey(config, "{currencyCode}")); + String currencyCode=currencyCodeColumn>=0? row.getCell(currencyCodeColumn).getStringCellValue(): "XOF"; if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) { if (!config.containsValue("{donorAgency}")) { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate, getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate, getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType, currencyCode); } else { int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType,currencyCode); } }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate,new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), separateFundingDate,new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType,currencyCode); } } @@ -439,9 +442,9 @@ private String getFundingDate(String yearString) return date.format(formatter); } private void updateFunding(ImportDataModel importDataModel, Session session, Number amount,String separateFundingDate, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String - adjustmentType) { + adjustmentType, String currencyCode) { String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; - Long currencyId = getCurrencyId(session); + Long currencyId = getCurrencyId(session,currencyCode); Long adjType = getCategoryValue(session, "adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY,catHql,adjustmentType ); Long assType = getCategoryValue(session, "assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, catHql,assistanceType); Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); @@ -506,7 +509,7 @@ private Long getOrganizationRole(Session session) { return orgRole; } - private Long getCurrencyId(Session session) { + private Long getCurrencyId(Session session, String currencyCode) { if (constantsMap.containsKey("currencyId")) { Long val = constantsMap.get("currencyId"); @@ -521,7 +524,7 @@ private Long getCurrencyId(Session session) { "WHERE ac.currencyCode = :currencyCode"; Query query = session.createQuery(hql); - query.setString("currencyCode", "XOF"); + query.setString("currencyCode", currencyCode); Long currencyId = (Long) query.uniqueResult(); constantsMap.put("currencyId", currencyId); return currencyId; @@ -788,6 +791,7 @@ private List getEntityFieldsInfo() { fieldsInfos.add("{financingInstrument}"); fieldsInfos.add("{typeOfAssistance}"); fieldsInfos.add("{secondarySubSector}"); + fieldsInfos.add("{currency}"); return fieldsInfos.stream().sorted().collect(Collectors.toList()); } From 53b83b3091764f16b56306cf49086d312c4bee55 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Wed, 5 Jun 2024 23:03:45 +0300 Subject: [PATCH 023/229] AMP-30885 Mapping Ecowas file with importer --- .../module/aim/action/DataImporter.java | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java index 4d22a37c41d..826992e76c8 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java @@ -26,7 +26,10 @@ import org.digijava.module.admin.dbentity.ImportedProject; import org.digijava.module.admin.util.ImportedFileUtil; import org.digijava.module.admin.util.model.*; -import org.digijava.module.aim.dbentity.*; +import org.digijava.module.aim.dbentity.AmpActivityVersion; +import org.digijava.module.aim.dbentity.AmpCurrency; +import org.digijava.module.aim.dbentity.AmpOrganisation; +import org.digijava.module.aim.dbentity.AmpRole; import org.digijava.module.aim.form.DataImporterForm; import org.digijava.module.aim.util.TeamMemberUtil; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; @@ -45,6 +48,7 @@ import java.sql.SQLException; import java.time.*; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -390,7 +394,6 @@ private void setAFundingItem(Sheet sheet, Map config, Row row, M String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - int separateFundingDateColumn=getColumnIndexByName(sheet, getKey(config, "{transactionDate}")); String separateFundingDate = separateFundingDateColumn>=0? row.getCell(separateFundingDateColumn).getStringCellValue(): null; int currencyCodeColumn=getColumnIndexByName(sheet, getKey(config, "{currencyCode}")); @@ -433,14 +436,60 @@ public static String findYearSubstring(String text) { } - private String getFundingDate(String yearString) + private String getFundingDate(String dateString) { - LocalDate date = LocalDate.of(Integer.parseInt(yearString), 1, 1); - + LocalDate date = LocalDate.now(); + if (isCommonDateFormat(dateString)){ + List formatters = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd"), + DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("MM/dd/yyyy"), + DateTimeFormatter.ofPattern("MM-dd-yyyy"), + DateTimeFormatter.ofPattern("yyyy/MM/dd"), + DateTimeFormatter.ofPattern("dd-MM-yyyy") + ); + + for (DateTimeFormatter formatter : formatters) { + try { + date = LocalDate.parse(dateString, formatter); + break; + } catch (DateTimeParseException e) { + // Continue to next formatter + } + } + } + else { + date = LocalDate.of(Integer.parseInt(dateString), 1, 1); + } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); return date.format(formatter); } + + + public static boolean isCommonDateFormat(String dateString) { + List dateFormats = Arrays.asList( + "yyyy-MM-dd", + "dd-MM-yyyy", + "MM-dd-yyyy", + "MM/dd/yyyy", + "dd/MM/yyyy", + "dd.MM.yyyy", + "yyyy/MM/dd" + ); + + for (String dateFormat : dateFormats) { + try { + LocalDate.parse(dateString, DateTimeFormatter.ofPattern(dateFormat)); + return true; + } catch (Exception e) { + // Ignore and continue with the next format + } + } + + return false; + } + private void updateFunding(ImportDataModel importDataModel, Session session, Number amount,String separateFundingDate, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String adjustmentType, String currencyCode) { String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; @@ -450,16 +499,24 @@ private void updateFunding(ImportDataModel importDataModel, Session session, Num Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); Long orgRole = getOrganizationRole(session); - String yearString; + String yearString = null; + String fundingDate; if (separateFundingDate!=null) { - yearString=findYearSubstring(separateFundingDate); + if (isCommonDateFormat(separateFundingDate)){ + fundingDate=getFundingDate(separateFundingDate); + }else { + yearString = findYearSubstring(separateFundingDate); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + + } } else { yearString=findYearSubstring(columnHeader); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + } - String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); Funding funding = new Funding(); funding.setDonor_organization_id(orgId); From 80eb0a478651014fb2f39ed33f983946ddd9df88 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 07:06:17 +0300 Subject: [PATCH 024/229] AMP-30885 Mapping Ecowas file with importer --- .../aim/action/dataimporter/ImporterUtil.java | 128 ++++++++++++++---- 1 file changed, 102 insertions(+), 26 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java index 69cdc146448..a08345b7886 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java @@ -35,6 +35,7 @@ import java.sql.SQLException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -48,53 +49,119 @@ public class ImporterUtil { private static final Logger logger = LoggerFactory.getLogger(ImporterUtil.class); static void setAFundingItemForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, String adjustmentType) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); - String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); - String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); + String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); + String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; + int separateFundingDateColumn=getColumnIndexByName(sheet, getKey(config, "{transactionDate}")); + String separateFundingDate = separateFundingDateColumn>=0? row.getCell(separateFundingDateColumn).getStringCellValue(): null; + int currencyCodeColumn=getColumnIndexByName(sheet, getKey(config, "{currencyCode}")); + String currencyCode=currencyCodeColumn>=0? row.getCell(currencyCodeColumn).getStringCellValue(): "XOF"; if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) { if (!config.containsValue("{donorAgency}")) { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType, currencyCode); } else { int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType, currencyCode); } }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType, currencyCode); } } + + private static String getFundingDate(String dateString) + { + LocalDate date = LocalDate.now(); + if (isCommonDateFormat(dateString)){ + List formatters = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd"), + DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("MM/dd/yyyy"), + DateTimeFormatter.ofPattern("MM-dd-yyyy"), + DateTimeFormatter.ofPattern("yyyy/MM/dd"), + DateTimeFormatter.ofPattern("dd-MM-yyyy") + ); + + for (DateTimeFormatter formatter : formatters) { + try { + date = LocalDate.parse(dateString, formatter); + break; + } catch (DateTimeParseException e) { + // Continue to next formatter + } + } + } + else { + date = LocalDate.of(Integer.parseInt(dateString), 1, 1); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + return date.format(formatter); + } + + + public static boolean isCommonDateFormat(String dateString) { + List dateFormats = Arrays.asList( + "yyyy-MM-dd", + "dd-MM-yyyy", + "MM-dd-yyyy", + "MM/dd/yyyy", + "dd/MM/yyyy", + "dd.MM.yyyy", + "yyyy/MM/dd" + ); + + for (String dateFormat : dateFormats) { + try { + LocalDate.parse(dateString, DateTimeFormatter.ofPattern(dateFormat)); + return true; + } catch (Exception e) { + // Ignore and continue with the next format + } + } + + return false; + } + static void setAFundingItemForTxt(Map row ,Map config, Map.Entry entry, ImportDataModel importDataModel, Session session,Number value, boolean commitment, boolean disbursement, String adjustmentType) { -// int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); String finInstrument= row.get(getKey(config, "{financingInstrument}")); finInstrument = finInstrument!= null? finInstrument : ""; - // detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); String typeOfAss =row.get(getKey(config, "{typeOfAssistance}")); typeOfAss=typeOfAss!=null? typeOfAss:""; + + + String separateFundingDate =row.get(getKey(config, "{transactionDate}")); + separateFundingDate=separateFundingDate!=null? separateFundingDate:""; + + String currencyCode =row.get(getKey(config, "{currencyCode}")); + currencyCode=currencyCode!=null? currencyCode:"XOF"; + + if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) { if (!config.containsValue("{donorAgency}")) { - updateFunding(importDataModel, session, value, entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, value, entry.getKey(),separateFundingDate, getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType, currencyCode); } else { String donorColumn = row.get(getKey(config, "{donorAgency}")); updateOrgs(importDataModel, donorColumn!=null && !donorColumn.isEmpty() ? donorColumn.trim():"no org", session, "donor"); - updateFunding(importDataModel, session, value, entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, value, entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType, currencyCode); } }else { - updateFunding(importDataModel, session, value, entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); + updateFunding(importDataModel, session, value, entry.getKey(),separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType,currencyCode); } } @@ -146,25 +213,34 @@ public static boolean isFileContentValid(File file) { } - private static String getFundingDate(String yearString) - { - LocalDate date = LocalDate.of(Integer.parseInt(yearString), 1, 1); - - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - return date.format(formatter); - } - private static void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String - adjustmentType) { + private static void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String separateFundingDate, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String + adjustmentType, String currencyCode) { String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; - Long currencyId = getCurrencyId(session); + Long currencyId = getCurrencyId(session,currencyCode); Long adjType = getCategoryValue(session, "adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY,catHql,adjustmentType ); Long assType = getCategoryValue(session, "assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, catHql,assistanceType); Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); Long orgRole = getOrganizationRole(session); - String yearString = findYearSubstring(columnHeader); - String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + String yearString = null; + String fundingDate; + if (separateFundingDate!=null) + { + if (isCommonDateFormat(separateFundingDate)){ + fundingDate=getFundingDate(separateFundingDate); + }else { + yearString = findYearSubstring(separateFundingDate); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + + } + } + else { + yearString=findYearSubstring(columnHeader); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + + } + Funding funding = new Funding(); funding.setDonor_organization_id(orgId); @@ -214,7 +290,7 @@ private static Long getOrganizationRole(Session session) { return orgRole; } - private static Long getCurrencyId(Session session) { + private static Long getCurrencyId(Session session, String currencyCode) { if (ConstantsMap.containsKey("currencyId")) { Long val = ConstantsMap.get("currencyId"); @@ -229,7 +305,7 @@ private static Long getCurrencyId(Session session) { "WHERE ac.currencyCode = :currencyCode"; Query query = session.createQuery(hql); - query.setString("currencyCode", "XOF"); + query.setString("currencyCode", currencyCode); Long currencyId = (Long) query.uniqueResult(); ConstantsMap.put("currencyId", currencyId); return currencyId; From e332b708a3331a40b1d24667fa4438457e45ecb2 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 07:07:17 +0300 Subject: [PATCH 025/229] AMP-30885 Mapping Ecowas file with importer --- .../module/aim/action/DataImporter.java | 781 ------------------ 1 file changed, 781 deletions(-) delete mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java deleted file mode 100644 index c66003a7ef5..00000000000 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/DataImporter.java +++ /dev/null @@ -1,781 +0,0 @@ -package org.digijava.module.aim.action; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; -import org.apache.poi.openxml4j.exceptions.InvalidOperationException; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.apache.struts.action.Action; -import org.apache.struts.action.ActionForm; -import org.apache.struts.action.ActionForward; -import org.apache.struts.action.ActionMapping; -import org.dgfoundation.amp.onepager.util.SessionUtil; -import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; -import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; -import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; -import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; -import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportStatus; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; -import org.digijava.module.admin.dbentity.ImportedProject; -import org.digijava.module.admin.util.ImportedFileUtil; -import org.digijava.module.admin.util.model.*; -import org.digijava.module.aim.dbentity.*; -import org.digijava.module.aim.form.DataImporterForm; -import org.digijava.module.aim.util.TeamMemberUtil; -import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; -import org.digijava.module.categorymanager.util.CategoryConstants; -import org.hibernate.Query; -import org.hibernate.Session; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.nio.file.Files; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.*; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.fasterxml.jackson.core.JsonGenerator.Feature.ESCAPE_NON_ASCII; - -public class DataImporter extends Action { - static Logger logger = LoggerFactory.getLogger(DataImporter.class); - private static final int BATCH_SIZE = 1000; - private static Map constantsMap=new HashMap<>(); - - - @Override - public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServletRequest request, HttpServletResponse response) throws Exception { - // List of fields - List fieldsInfo = getEntityFieldsInfo(); - request.setAttribute("fieldsInfo", fieldsInfo); - DataImporterForm dataImporterForm = (DataImporterForm)form; - - - if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { - logger.info(" this is the action "+request.getParameter("action")); - - InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - Set headersSet = new HashSet<>(); - for (int i=0;i cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); - } - - } - headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); - StringBuilder headers = new StringBuilder(); - headers.append(" \n"); - response.setHeader("selectTag",headers.toString()); - - - dataImporterForm.getColumnPairs().clear(); - - workbook.close(); - return null; - } - - - - - if (Objects.equals(request.getParameter("action"), "addField")) { - logger.info(" this is the action "+request.getParameter("action")); - - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - logger.info("Column Pairs:"+dataImporterForm.getColumnPairs()); - - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString( dataImporterForm.getColumnPairs()); - - // Send JSON response - response.setContentType("application/json"); - response.getWriter().write(json); - response.setCharacterEncoding("UTF-8"); - - return null; - - } - - - if (Objects.equals(request.getParameter("action"), "removeField")) { - logger.info(" this is the action "+request.getParameter("action")); - - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - removeMapItem(dataImporterForm.getColumnPairs(),columnName,selectedField); - logger.info("Column Pairs:"+dataImporterForm.getColumnPairs()); - - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString( dataImporterForm.getColumnPairs()); - - // Send JSON response - response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); -// response.setHeader("updatedMap",json); - response.getWriter().write(json); - - return null; - - } - - if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { - logger.info("This is the action Upload " + request.getParameter("uploadDataFile")); - String fileName = dataImporterForm.getDataFile().getFileName(); - String tempDirPath = System.getProperty("java.io.tmpdir"); - File tempDir = new File(tempDirPath); - if (!tempDir.exists()) { - tempDir.mkdirs(); - } - String tempFilePath = tempDirPath + File.separator + fileName; - try (InputStream inputStream = dataImporterForm.getDataFile().getInputStream(); - FileOutputStream outputStream = new FileOutputStream(tempFilePath)) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - } - - // Check if the file is readable and has correct content - File tempFile = new File(tempFilePath); - List similarFiles = ImportedFileUtil.getSimilarFiles(tempFile); - if (similarFiles!=null && !similarFiles.isEmpty()) { - for (ImportedFilesRecord similarFilesRecord : similarFiles) { - logger.info("Similar file: " + similarFilesRecord); - if (similarFilesRecord.getImportStatus().equals(ImportStatus.IN_PROGRESS)) - { - response.setHeader("errorMessage","You have a similar file in progress. Please try again later."); - response.setStatus(400); - return mapping.findForward("importData"); - } - } - } - if (dataImporterForm.getColumnPairs().isEmpty() || !dataImporterForm.getColumnPairs().containsValue("{projectTitle}")) - { - response.setHeader("errorMessage","You must have atleast the {projectTitle} key in your config."); - response.setStatus(400); - return mapping.findForward("importData"); - } - if (!isFileReadable(tempFile) || !isFileContentValid(tempFile)) { - // Handle invalid file - logger.error("Invalid file or content."); - response.setHeader("errorMessage","Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); - - // Optionally, you can respond with an error message to the client. - } else { - // Proceed with processing the file - ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); - // Process the file in batches - int res = processFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs()); - if (res != 1) { - // Handle error - logger.info("Error processing file " + tempFile); - response.setHeader("errorMessage","Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); - } - - // Clean up - Files.delete(tempFile.toPath()); - logger.info("Cache map size: " + constantsMap.size()); - constantsMap.clear(); - logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); - Instant start = Instant.now(); - logger.info("Start time: " + start); - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - logger.info("Time Elapsed: " + timeElapsed); - - // Send response - response.setHeader("updatedMap", ""); - dataImporterForm.getColumnPairs().clear(); - } - return null; - } - - return mapping.findForward("importData"); - } - - private void removeMapItem(Map map,String columnName, String selectedField) - { - // Check if the entry's key and value match the criteria - // Remove the entry - map.entrySet().removeIf(entry -> columnName.equals(entry.getKey()) && selectedField.equals(entry.getValue())); - } - - - public static boolean isFileReadable(File file) { - if (file == null || !file.exists() || !file.isFile()) { - return false; - } - return file.canRead(); - } - - // Check if the file content is valid - public static boolean isFileContentValid(File file) { - // Define your validation criteria here - // For example, let's say we want to check if the file contains at least one line - try (BufferedReader reader = new BufferedReader(new FileReader(file))) { - String line = reader.readLine(); - return line != null; // If at least one line exists, consider the content valid - } catch (IOException e) { - e.printStackTrace(); // Handle the exception appropriately - return false; // Consider the content invalid if an exception occurs - } - } - - public int processFileInBatches(ImportedFilesRecord importedFilesRecord,File file, HttpServletRequest request,Map config) { - // Open the workbook - int res=0; - ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); - try (Workbook workbook = new XSSFWorkbook(file)) { - int numberOfSheets = workbook.getNumberOfSheets(); - logger.info("Number of sheets: " + numberOfSheets); - - // Process each sheet in the workbook - for (int i = 0; i < numberOfSheets; i++) { - logger.info("Sheet number: " + i); - Sheet sheet = workbook.getSheetAt(i); - processSheetInBatches(sheet, request,config, importedFilesRecord); - } - - logger.info("Closing the workbook..."); - ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.SUCCESS); - res =1; - } catch (IOException e) { - ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); - logger.error("Error processing Excel file: " + e.getMessage(), e); - } catch (InvalidFormatException | InvalidOperationException e) { - logger.error("Error processing Excel file: " + e.getMessage(),e); - } - logger.info("Finished processing file record id: "+importedFilesRecord.getId()+" with status: "+importedFilesRecord.getImportStatus()); - return res; - - } - - private void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { - // Get the number of rows in the sheet - int rowCount = sheet.getPhysicalNumberOfRows(); - logger.info("Total number of rows: " + rowCount); - - // Process each row in batches - for (int i = 0; i < rowCount; i += BATCH_SIZE) { - int endIndex = Math.min(i + BATCH_SIZE, rowCount); - List batch = new ArrayList<>(); - - // Retrieve a batch of rows - for (int j = i; j < endIndex; j++) { - Row row = sheet.getRow(j); - if (row != null) { - if (row.getRowNum() == 0) { - continue; - } - batch.add(row); - } - } - - // Process the batch - processBatch(batch, sheet, request,config, importedFilesRecord); - } - } - - private void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { - // Process the batch of rows - SessionUtil.extendSessionIfNeeded(request); - Session session = PersistenceManager.getRequestDBSession(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - for (Row row : batch) { - ImportedProject importedProject= new ImportedProject(); - importedProject.setImportedFilesRecord(importedFilesRecord); - - ImportDataModel importDataModel = new ImportDataModel(); - importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); - importDataModel.setIs_draft(true); - OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); - importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel, session); - - logger.info("Row Number: "+row.getRowNum()+", Sheet Name: "+sheet.getSheetName()); - - for (Map.Entry entry : config.entrySet()) { - int columnIndex = getColumnIndexByName(sheet, entry.getKey()); - if (columnIndex >= 0) { - Cell cell = row.getCell(columnIndex); - switch (entry.getValue()) { - case "{projectTitle}": - importDataModel.setProject_title(cell.getStringCellValue().trim()); - break; - case "{projectDescription}": - importDataModel.setDescription(cell.getStringCellValue().trim()); - break; - case "{projectLocation}": -// ampActivityVersion.addLocation(new AmpActivityLocation()); - break; - case "{primarySector}": - updateSectors(importDataModel, cell.getStringCellValue().trim(), session, true); - break; - case "{secondarySector}": - updateSectors(importDataModel, cell.getStringCellValue().trim(), session, false); - break; - case "{donorAgency}": - updateOrgs(importDataModel, cell.getStringCellValue().trim(), session, "donor"); - break; - case "{fundingItem}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,true, "Actual"); - break; - case "{plannedCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Planned"); - break; - case "{plannedDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Planned"); - break; - case "{actualCommitment}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,true,false, "Actual"); - break; - case "{actualDisbursement}": - setAFundingItem(sheet, config, row, entry, importDataModel, session, cell,false,true, "Actual"); - break; - default: - logger.error("Unexpected value: " + entry.getValue()); - break; - } - - - - } - } - importTheData(importDataModel, session, importedProject); - - } - } - - private void setAFundingItem(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell,boolean commitment, boolean disbursement, String - adjustmentType) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "{financingInstrument}")); - String finInstrument= detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "{typeOfAssistance}")); - String typeOfAss = detailColumn>=0? row.getCell(detailColumn).getStringCellValue(): ""; - - if (importDataModel.getDonor_organization()==null || importDataModel.getDonor_organization().isEmpty()) - { - if (!config.containsValue("{donorAgency}")) - { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), getRandomOrg(session),typeOfAss,finInstrument, commitment,disbursement, adjustmentType); - - } - else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "{donorAgency}")); - updateOrgs(importDataModel, columnIndex1>=0? row.getCell(columnIndex1).getStringCellValue().trim():"no org", session, "donor"); - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); - } - - }else { - updateFunding(importDataModel, session, cell.getNumericCellValue(), entry.getKey(), new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(),typeOfAss,finInstrument,commitment,disbursement, adjustmentType); - } - } - - public K getKey(Map map, V value) { - for (Map.Entry entry : map.entrySet()) { - if (entry.getValue().equals(value)) { - return entry.getKey(); - } - } - return null; - } - - public static String findYearSubstring(String text) { - Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); - Matcher matcher = pattern.matcher(text); - if (matcher.find()) { - return matcher.group(); - } else { - return null; - } - } - - - private String getFundingDate(String yearString) - { - LocalDate date = LocalDate.of(Integer.parseInt(yearString), 1, 1); - - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - return date.format(formatter); - } - private void updateFunding(ImportDataModel importDataModel, Session session, Number amount, String columnHeader, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, String - adjustmentType) { - String catHql="SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; - Long currencyId = getCurrencyId(session); - Long adjType = getCategoryValue(session, "adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY,catHql,adjustmentType ); - Long assType = getCategoryValue(session, "assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, catHql,assistanceType); - Long finInstrument = getCategoryValue(session, "finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, catHql,finInst); - Long orgRole = getOrganizationRole(session); - - String yearString = findYearSubstring(columnHeader); - String fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); - - Funding funding = new Funding(); - funding.setDonor_organization_id(orgId); - funding.setType_of_assistance(assType); - funding.setFinancing_instrument(finInstrument); - funding.setSource_role(orgRole); - - Transaction transaction = new Transaction(); - transaction.setCurrency(currencyId); - transaction.setAdjustment_type(adjType); - transaction.setTransaction_amount(amount.doubleValue()); - transaction.setTransaction_date(fundingDate); - if (commitment) { - funding.getCommitments().add(transaction); - } - if (disbursement) { - funding.getDisbursements().add(transaction); - } - - DonorOrganization donorOrganization = new DonorOrganization(); - donorOrganization.setOrganization(orgId); - donorOrganization.setPercentage(100.0); - - importDataModel.getDonor_organization().add(donorOrganization); - importDataModel.getFundings().add(funding); - } - - - private Long getOrganizationRole(Session session) { - - if (constantsMap.containsKey("orgRole")) { - Long val= constantsMap.get("orgRole"); - logger.info("In cache... orgRole: "+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampRoleId FROM " + AmpRole.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; - - Query query = session.createQuery(hql); - query.setParameter("name", "%donor%"); - List orgRoles = query.list(); - Long orgRole = orgRoles.get(0); - constantsMap.put("orgRole", orgRole); - return orgRole; - } - - private Long getCurrencyId(Session session) { - - if (constantsMap.containsKey("currencyId")) { - Long val = constantsMap.get("currencyId"); - logger.info("In cache... currency: "+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT ac.ampCurrencyId FROM " + AmpCurrency.class.getName() + " ac " + - "WHERE ac.currencyCode = :currencyCode"; - - Query query = session.createQuery(hql); - query.setString("currencyCode", "XOF"); - Long currencyId = (Long) query.uniqueResult(); - constantsMap.put("currencyId", currencyId); - return currencyId; - } - - private Long getCategoryValue(Session session, String constantKey, String categoryKey, String hql, String possibleValue) { - String fullKey=constantKey+"_"+possibleValue; - if (constantsMap.containsKey(fullKey)) { - Long val = constantsMap.get(fullKey); - logger.info("In cache... "+fullKey+":"+val); - return val; - - } - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - Query query = session.createQuery(hql); - query.setParameter("categoryKey", categoryKey); - List values = query.list(); - Long categoryId = ((AmpCategoryValue) values.get(0)).getId(); - - if (!Objects.equals(possibleValue, "") && !Objects.equals(possibleValue, null)) - { - for (Object categoryValue : values) - { - if (Objects.equals(((AmpCategoryValue) categoryValue).getValue().toLowerCase(), possibleValue.toLowerCase())) - { - categoryId = ((AmpCategoryValue) categoryValue).getId(); - logger.info("Found category: "+((AmpCategoryValue) categoryValue).getValue()); - break; - } - - } - } - logger.info("Found category: "+categoryId +" for "+constantKey+"_"+possibleValue); - constantsMap.put(fullKey, categoryId); - return categoryId; - } - - private AmpActivityVersion existingActivity(ImportDataModel importDataModel,Session session) - { - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + - "WHERE a.name = :name"; - Query query= session.createQuery(hql); - query.setString("name", importDataModel.getProject_title()); - List ampActivityVersions=query.list(); - return !ampActivityVersions.isEmpty()? ampActivityVersions.get(ampActivityVersions.size()-1) :null; - } - private void setStatus(ImportDataModel importDataModel,Session session) - { - String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s " + - "JOIN s.ampCategoryClass c " + - "WHERE c.keyName = :categoryKey"; - Long statusId = getCategoryValue(session,"statusId",CategoryConstants.ACTIVITY_STATUS_KEY,hql,""); - importDataModel.setActivity_status(statusId); - - } - private void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject) throws JsonProcessingException { - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - ActivityImportRules rules = new ActivityImportRules(true, false, - true); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization - objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); - - Map map = objectMapper - .convertValue(importDataModel, new TypeReference>() {}); - JsonApiResponse response; - AmpActivityVersion existing = existingActivity(importDataModel,session); - logger.info("Existing ?"+existing); - logger.info("Data model object: "+importDataModel); - if (existing==null){ - logger.info("New activity"); - importedProject.setNewProject(true); - response= ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); - } - else - { - logger.info("Existing activity"); - importedProject.setNewProject(false); - importDataModel.setInternal_id(existing.getAmpActivityId()); - importDataModel.setAmp_id(existing.getAmpId()); - ActivityGroup activityGroup= new ActivityGroup(); - activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); - importDataModel.setActivity_group(activityGroup); - map = objectMapper - .convertValue(importDataModel, new TypeReference>() {}); - response= ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); - - } - if (response!=null) { - if (!response.getErrors().isEmpty()) { - importedProject.setImportStatus(ImportStatus.FAILED); - } else { - importedProject.setImportStatus(ImportStatus.SUCCESS); - - } - } - - String resp = objectMapper.writeValueAsString(response); - importedProject.setImportResponse(resp); - if (!session.isOpen()) { - session=PersistenceManager.getRequestDBSession(); - } - session.saveOrUpdate(importedProject); - logger.info("Imported project: "+importedProject); - } - - - - private void updateSectors(ImportDataModel importDataModel, String name, Session session, boolean primary) - { - - if (constantsMap.containsKey("sector_"+name)) { - Long sectorId = constantsMap.get("sector_"+name); - logger.info("In cache... sector "+"sector_"+name+":"+sectorId); - createSector(importDataModel,primary,sectorId); - } - else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - - session.doWork(connection -> { - String query = primary ? "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Primary'" : "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Secondary'"; - try (PreparedStatement statement = connection.prepareStatement(query)) { - // Set the name as a parameter to the prepared statement - statement.setString(1, name); - - // Execute the query and process the results - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long ampSectorId = resultSet.getLong("amp_sector_id"); - createSector(importDataModel, primary, ampSectorId); - constantsMap.put("sector_"+name, ampSectorId); - } - } - - } catch (SQLException e) { - logger.error("Error getting sectors", e); - } - }); - } - - - - } - - private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { - Sector sector1 = new Sector(); - sector1.setSector_percentage(100.00); - sector1.setSector(ampSectorId); - if (primary) { - importDataModel.getPrimary_sectors().add(sector1); - } - else - { - importDataModel.getSecondary_sectors().add(sector1); - - } - } - - private Long getRandomOrg(Session session) - { - Long randomOrg; - if (constantsMap.containsKey("randomOrg")) { - randomOrg = constantsMap.get("randomOrg"); - logger.info("In cache... randomOrg "+randomOrg); - }else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; - - randomOrg = (Long) session.createQuery(hql).setMaxResults(1).uniqueResult(); - constantsMap.put("randomOrg",randomOrg); - } - return randomOrg; - - - } - - private void updateOrgs(ImportDataModel importDataModel, String name, Session session, String type) - { - Long orgId; - - if (constantsMap.containsKey("org_"+name)) { - orgId = constantsMap.get("org_"+name); - logger.info("In cache... organisation "+"org_"+name+":"+orgId); - } - else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - String hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.name) LIKE LOWER(:name)"; - - Query query = session.createQuery(hql); - query.setParameter("name", "%" + name + "%"); - List organisations = query.list(); - if (!organisations.isEmpty()) { - orgId = organisations.get(0); - } else { - hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o"; - - query = session.createQuery(hql).setMaxResults(1); - orgId = (Long) query.uniqueResult(); - } - constantsMap.put("org_" + name, orgId); - } - logger.info("Organisation: " + orgId); - - if (Objects.equals(type, "donor")) { - DonorOrganization donorOrganization = new DonorOrganization(); - donorOrganization.setOrganization(orgId); - donorOrganization.setPercentage(100.0); - importDataModel.getDonor_organization().add(donorOrganization); - } - - - - } - - - private static int getColumnIndexByName(Sheet sheet, String columnName) { - try { - Row headerRow = sheet.getRow(0); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - if (cell != null && columnName.equals(cell.getStringCellValue())) { - return i; - } - } - return -1; - }catch (Exception e) - { - logger.error("Error getting column index for "+columnName,e); - return -1; - } - - } - private List getEntityFieldsInfo() { - List fieldsInfos = new ArrayList<>(); - fieldsInfos.add("{projectTitle}"); - fieldsInfos.add("{projectDescription}"); - fieldsInfos.add("{primarySector}"); - fieldsInfos.add("{secondarySector}"); - fieldsInfos.add("{projectLocation}"); - fieldsInfos.add("{projectStartDate}"); - fieldsInfos.add("{projectEndDate}"); - fieldsInfos.add("{donorAgency}"); - fieldsInfos.add("{executingAgency}"); - fieldsInfos.add("{implementingAgency}"); - fieldsInfos.add("{actualDisbursement}"); - fieldsInfos.add("{actualCommitment}"); - fieldsInfos.add("{plannedDisbursement}"); - fieldsInfos.add("{plannedCommitment}"); - fieldsInfos.add("{fundingItem}"); - fieldsInfos.add("{financingInstrument}"); - fieldsInfos.add("{typeOfAssistance}"); - fieldsInfos.add("{secondarySubSector}"); - return fieldsInfos.stream().sorted().collect(Collectors.toList()); - } - -} From 8c66c1b7b1dc22aa7a39d7d521ab34c4e29d4676 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 08:06:06 +0300 Subject: [PATCH 026/229] AMP-30885 Mapping Ecowas file with importer(txt) --- .../digijava/module/admin/util/DbUtil.java | 10 ++---- .../module/aim/action/ViewImportProgress.java | 6 ++-- .../aim/action/dataimporter/DataImporter.java | 10 +++--- .../aim/action/dataimporter/ImporterUtil.java | 5 +-- .../action/dataimporter/TxtDataImporter.java | 6 ++-- .../dbentity/DataImporterConfig.hbm.xml | 19 ++++++++++ .../dbentity/DataImporterConfig.java | 35 +++++++++++++++++++ .../dbentity/DataImporterConfigValues.hbm.xml | 15 ++++++++ .../dbentity/DataImporterConfigValues.java | 33 +++++++++++++++++ .../dataimporter}/dbentity/ImportStatus.java | 2 +- .../dbentity/ImportedFilesRecord.hbm.xml | 4 +-- .../dbentity/ImportedFilesRecord.java | 2 +- .../dbentity/ImportedProject.hbm.xml | 6 ++-- .../dbentity/ImportedProject.java | 2 +- .../dataimporter}/model/ImportDataModel.java | 4 ++- .../dataimporter}/util/ImportedFileUtil.java | 6 ++-- .../module/aim/form/ImportProgressForm.java | 4 +-- amp/deployConfigs/selected.properties | 2 +- amp/repository/digi-common.xml | 6 ++-- 19 files changed, 140 insertions(+), 37 deletions(-) create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.hbm.xml create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.java create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.hbm.xml create mode 100644 amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.java rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/dbentity/ImportStatus.java (51%) rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/dbentity/ImportedFilesRecord.hbm.xml (81%) rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/dbentity/ImportedFilesRecord.java (94%) rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/dbentity/ImportedProject.hbm.xml (75%) rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/dbentity/ImportedProject.java (96%) rename amp/WEB-INF/src/org/digijava/module/{admin/util => aim/action/dataimporter}/model/ImportDataModel.java (99%) rename amp/WEB-INF/src/org/digijava/module/{admin => aim/action/dataimporter}/util/ImportedFileUtil.java (92%) diff --git a/amp/WEB-INF/src/org/digijava/module/admin/util/DbUtil.java b/amp/WEB-INF/src/org/digijava/module/admin/util/DbUtil.java index 4fb231901db..8c4c72a3054 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/util/DbUtil.java +++ b/amp/WEB-INF/src/org/digijava/module/admin/util/DbUtil.java @@ -22,12 +22,6 @@ package org.digijava.module.admin.util; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.StringTokenizer; - import org.apache.log4j.Logger; import org.dgfoundation.amp.error.AMPException; import org.digijava.kernel.ampapi.endpoints.config.utils.ConfigHelper; @@ -46,6 +40,8 @@ import org.hibernate.Session; import org.hibernate.Transaction; +import java.util.*; + public class DbUtil { private static Logger logger = Logger.getLogger(DbUtil.class); @@ -200,7 +196,7 @@ public static Group getGroupByKey(String key) throws logger.debug("Unable to get group from database ", ex); throw new AdminException("Unable to get group from database ", ex); } - return group; + return group; } public static void editGroup(Group group) throws AdminException { Session sess = null; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/ViewImportProgress.java b/amp/WEB-INF/src/org/digijava/module/aim/action/ViewImportProgress.java index b560ce4913b..8c2ae7cbc17 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/ViewImportProgress.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/ViewImportProgress.java @@ -7,9 +7,9 @@ import org.apache.struts.action.ActionForward; import org.apache.struts.action.ActionMapping; import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportStatus; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; -import org.digijava.module.admin.dbentity.ImportedProject; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.form.ImportProgressForm; import org.hibernate.Query; import org.hibernate.Session; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index 457a2d5cf02..e601c7b15c4 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -20,11 +20,11 @@ import org.apache.struts.action.ActionMapping; import org.dgfoundation.amp.onepager.util.SessionUtil; import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportStatus; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; -import org.digijava.module.admin.dbentity.ImportedProject; -import org.digijava.module.admin.util.ImportedFileUtil; -import org.digijava.module.admin.util.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; +import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; import org.digijava.module.aim.form.DataImporterForm; import org.digijava.module.aim.util.TeamMemberUtil; import org.hibernate.Session; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java index a08345b7886..3c6a8f00e46 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/ImporterUtil.java @@ -12,9 +12,10 @@ import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportStatus; -import org.digijava.module.admin.dbentity.ImportedProject; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.admin.util.model.*; +import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.dbentity.AmpCurrency; import org.digijava.module.aim.dbentity.AmpOrganisation; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index d6b7932cb03..9c13d2d2945 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -8,9 +8,9 @@ import com.opencsv.exceptions.CsvValidationException; import org.dgfoundation.amp.onepager.util.SessionUtil; import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; -import org.digijava.module.admin.dbentity.ImportedProject; -import org.digijava.module.admin.util.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; +import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; import org.digijava.module.aim.util.TeamMemberUtil; import org.hibernate.Session; import org.slf4j.Logger; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.hbm.xml b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.hbm.xml new file mode 100644 index 00000000000..6368243a9df --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.hbm.xml @@ -0,0 +1,19 @@ + + + + + + + DATA_IMPORTER_CONFIG_SEQ + + + + + + + + + + + + diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.java new file mode 100644 index 00000000000..07faedfa811 --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfig.java @@ -0,0 +1,35 @@ +package org.digijava.module.aim.action.dataimporter.dbentity; + +import java.io.Serializable; +import java.util.Map; + +public class DataImporterConfig implements Serializable { + private Long id; + private String configName; + private Map configValues; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getConfigName() { + return configName; + } + + public void setConfigName(String configName) { + this.configName = configName; + } + + public Map getConfigValues() { + return configValues; + } + + public void setConfigValues(Map configValues) { + this.configValues = configValues; + } +} diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.hbm.xml b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.hbm.xml new file mode 100644 index 00000000000..f6b00115962 --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.hbm.xml @@ -0,0 +1,15 @@ + + + + + + + DATA_IMPORTER_CONFIG_VALUES_SEQ + + + + + + + + diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.java new file mode 100644 index 00000000000..8d6679ade1c --- /dev/null +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/DataImporterConfigValues.java @@ -0,0 +1,33 @@ +package org.digijava.module.aim.action.dataimporter.dbentity; + +import java.io.Serializable; + +public class DataImporterConfigValues implements Serializable { + private String configKey; + private String configValue; + private Long id; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getConfigKey() { + return configKey; + } + + public void setConfigKey(String configKey) { + this.configKey = configKey; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } +} diff --git a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportStatus.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportStatus.java similarity index 51% rename from amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportStatus.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportStatus.java index 602ed77d3a7..18c10952ace 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportStatus.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportStatus.java @@ -1,4 +1,4 @@ -package org.digijava.module.admin.dbentity; +package org.digijava.module.aim.action.dataimporter.dbentity; public enum ImportStatus { UPLOADED,IN_PROGRESS,SUCCESS,FAILED diff --git a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.hbm.xml b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml similarity index 81% rename from amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.hbm.xml rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml index ca4e4d8f78f..1b4d8649f18 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.hbm.xml +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml @@ -2,7 +2,7 @@ - @@ -14,7 +14,7 @@ - + diff --git a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java similarity index 94% rename from amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java index a8087c06004..6a9a872a3fd 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedFilesRecord.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java @@ -1,4 +1,4 @@ -package org.digijava.module.admin.dbentity; +package org.digijava.module.aim.action.dataimporter.dbentity; import java.io.Serializable; diff --git a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.hbm.xml b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.hbm.xml similarity index 75% rename from amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.hbm.xml rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.hbm.xml index a1080a7fa13..b244a68632a 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.hbm.xml +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.hbm.xml @@ -2,7 +2,7 @@ - @@ -10,12 +10,12 @@ IMPORTED_PROJECT_SEQ - + - + diff --git a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.java similarity index 96% rename from amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.java index 7d95157368e..1db5d34b344 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/dbentity/ImportedProject.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProject.java @@ -1,4 +1,4 @@ -package org.digijava.module.admin.dbentity; +package org.digijava.module.aim.action.dataimporter.dbentity; public class ImportedProject { private Long id; diff --git a/amp/WEB-INF/src/org/digijava/module/admin/util/model/ImportDataModel.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java similarity index 99% rename from amp/WEB-INF/src/org/digijava/module/admin/util/model/ImportDataModel.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java index 1215b38aaad..4212fa86529 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/util/model/ImportDataModel.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java @@ -1,4 +1,6 @@ -package org.digijava.module.admin.util.model; +package org.digijava.module.aim.action.dataimporter.model; + +import org.digijava.module.admin.util.model.*; import java.util.HashSet; import java.util.Set; diff --git a/amp/WEB-INF/src/org/digijava/module/admin/util/ImportedFileUtil.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java similarity index 92% rename from amp/WEB-INF/src/org/digijava/module/admin/util/ImportedFileUtil.java rename to amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java index 9705f8940f2..3d201aaf3eb 100644 --- a/amp/WEB-INF/src/org/digijava/module/admin/util/ImportedFileUtil.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java @@ -1,8 +1,8 @@ -package org.digijava.module.admin.util; +package org.digijava.module.aim.action.dataimporter.util; import org.digijava.kernel.persistence.PersistenceManager; -import org.digijava.module.admin.dbentity.ImportStatus; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.hibernate.Query; import org.hibernate.Session; import org.slf4j.Logger; diff --git a/amp/WEB-INF/src/org/digijava/module/aim/form/ImportProgressForm.java b/amp/WEB-INF/src/org/digijava/module/aim/form/ImportProgressForm.java index e2fc12071af..3f759831be2 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/form/ImportProgressForm.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/form/ImportProgressForm.java @@ -1,8 +1,8 @@ package org.digijava.module.aim.form; import org.apache.struts.action.ActionForm; -import org.digijava.module.admin.dbentity.ImportedFilesRecord; -import org.digijava.module.admin.dbentity.ImportedProject; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; +import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import java.util.ArrayList; import java.util.List; diff --git a/amp/deployConfigs/selected.properties b/amp/deployConfigs/selected.properties index c9e4a92ccf1..16a303528e2 100644 --- a/amp/deployConfigs/selected.properties +++ b/amp/deployConfigs/selected.properties @@ -1,4 +1,4 @@ #Description of the selected config -#Wed, 13 Sep 2023 13:20:42 +0300 +#Wed, 05 Jun 2024 23:08:45 +0300 serverName=local diff --git a/amp/repository/digi-common.xml b/amp/repository/digi-common.xml index beac7714e64..450f175d2f8 100644 --- a/amp/repository/digi-common.xml +++ b/amp/repository/digi-common.xml @@ -48,8 +48,10 @@ org.digijava.module.common.dbentity.ItemStatus org.digijava.module.common.dbentity.ModuleTeaser org.digijava.kernel.entity.MailSpool - org.digijava.module.admin.dbentity.ImportedFilesRecord - org.digijava.module.admin.dbentity.ImportedProject + org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord + org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject + org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfig + org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfigValues From 6507c74104eca65a40c5ad88bc05e8757e32f5b6 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 08:58:21 +0300 Subject: [PATCH 027/229] AMP-30885 Mapping Ecowas file with importer(txt) --- .../aim/action/dataimporter/DataImporter.java | 94 +++++++++++++++++-- amp/repository/aim/view/dataImporter.jsp | 70 +++++++++++--- 2 files changed, 144 insertions(+), 20 deletions(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index e601c7b15c4..9e3e90b3087 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -20,13 +20,15 @@ import org.apache.struts.action.ActionMapping; import org.dgfoundation.amp.onepager.util.SessionUtil; import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfig; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; -import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; import org.digijava.module.aim.form.DataImporterForm; import org.digijava.module.aim.util.TeamMemberUtil; +import org.hibernate.Query; import org.hibernate.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,10 +37,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.*; import java.nio.file.Files; -import java.time.Duration; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; +import java.time.*; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -55,10 +54,31 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet // List of fields List fieldsInfo = getEntityFieldsInfo(); request.setAttribute("fieldsInfo", fieldsInfo); + List configNames= getConfigNames(); + request.setAttribute("configNames", configNames); DataImporterForm dataImporterForm = (DataImporterForm) form; + if (Objects.equals(request.getParameter("action"), "configByName")) { + logger.info(" this is the action " + request.getParameter("action")); + String configName = request.getParameter("configName"); + Map config= getConfigByName(configName); + dataImporterForm.setColumnPairs(config); + + logger.info("Column Pairs:" + config); + + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(config); + + // Send JSON response + response.setContentType("application/json"); + response.getWriter().write(json); + response.setCharacterEncoding("UTF-8"); + + return null; + } + - if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { + if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { logger.info(" this is the action " + request.getParameter("action")); if (request.getParameter("uploadTemplate") != null) { logger.info(" this is the action " + request.getParameter("uploadTemplate")); @@ -154,7 +174,6 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet // Send JSON response response.setContentType("application/json"); response.setCharacterEncoding("UTF-8"); -// response.setHeader("updatedMap",json); response.getWriter().write(json); return null; @@ -165,6 +184,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info("This is the action " + request.getParameter("action")); String fileName = dataImporterForm.getDataFile().getFileName(); String tempDirPath = System.getProperty("java.io.tmpdir"); + saveImportConfig(request, fileName,dataImporterForm.getColumnPairs()); File tempDir = new File(tempDirPath); if (!tempDir.exists()) { tempDir.mkdirs(); @@ -246,6 +266,66 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet return mapping.findForward("importData"); } + private static List getConfigNames() + { + Session session = PersistenceManager.getRequestDBSession(); + + if (!session.isOpen()) { + session=PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT c.configName FROM DataImporterConfig c"; + Query query = session.createQuery(hql); + + // Execute query and get the list of configNames + List< String> configNames = query.list(); + return configNames==null?Collections.emptyList():configNames; + + } + + private static Map getConfigByName(String configName) { + logger.info("Getting import config"); + Session session = PersistenceManager.getRequestDBSession(); + + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + + Query query = session.createQuery("d.configValues FROM DataImporterConfig d WHERE configName = :configName"); + query.setParameter("configName", configName); + query.setMaxResults(1); + + // Add return statement + return (Map) query.list().get(0); +} + private static void saveImportConfig(HttpServletRequest request,String fileName, Map config) + { + logger.info("Saving import config"); + Session session = PersistenceManager.getRequestDBSession(); + + if (!session.isOpen()) { + session=PersistenceManager.getRequestDBSession(); + } + String configName= fileName+"_"+ LocalDateTime.now().toString().replace(":", "_"); + + if (request.getParameter("configName") != null) + { + configName=request.getParameter("configName"); + Query query = session.createQuery("FROM DataImporterConfig WHERE configName = :configName"); + query.setParameter("configName", configName); + List existingConfigs = query.list(); + + if (!existingConfigs.isEmpty()) { + configName += "_" + LocalDateTime.now().toString().replace(":", "_"); + } + } + + DataImporterConfig dataImporterConfig= new DataImporterConfig(); + dataImporterConfig.setConfigValues(config); + dataImporterConfig.setConfigName(configName); + session.saveOrUpdate(dataImporterConfig); + logger.info("Saved configuration"); + + } diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index e1d554e16f6..6eb50a37579 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -28,14 +28,6 @@ } $(document).ready(function() { - // $('.fields-table tbody').on('click', '.remove-field', function() { - // console.log("Removing..."); - // var $row = $(this).closest('tr'); - // var selectedField = $row.find('.selected-field').text(); - // var columnName = $row.find('.column-name').text(); - // sendValuesToBackend(columnName,selectedField,"removeField"); - // - // }); $('.remove-row').click(function() { var selectedRows = $('.fields-table tbody').find('.remove-checkbox:checked').closest('tr'); @@ -82,6 +74,50 @@ $('#separator-div').hide(); } }); + + $('.existing-config').change(function() { + var configName = $(this).val(); + + var formData = new FormData(); + formData.append("configName", configName); + formData.append("action", "configByName"); + + fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { + method: "POST", + body: formData + }) + .then(response =>{ + if (!response.ok) { + throw new Error("Network response was not ok"); + } + // console.log("Response: ",response.json()); + + return response.json(); + }) + .then(updatedMap => { + console.log("Map :" ,updatedMap) + + // Update UI or perform any additional actions if needed + console.log("Selected pairs updated successfully."); + console.log("Updated map received:", updatedMap); + var tbody = document.getElementById("selected-pairs-table-body"); + + // Remove all rows from the table body + tbody.innerHTML = ""; + + for (var key in updatedMap) { + if (updatedMap.hasOwnProperty(key)) { + // Access each property using the key + var value = updatedMap[key]; + updateTable(key, value, tbody); + console.log('Key:', key, 'Value:', value); + } + } + }) + .catch(error => { + console.error("There was a problem with the fetch operation:", error); + }); + }); }); @@ -247,6 +283,18 @@

Data Importer

+ + + + +

Upload File

Data file configuration

@@ -336,11 +384,7 @@ - + From 186c60bed38ffc31ca4f7e0869441367d19bc61f Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 09:04:03 +0300 Subject: [PATCH 028/229] AMP-30885 Mapping Ecowas file with importer(txt) --- .../digijava/module/aim/action/dataimporter/DataImporter.java | 2 +- amp/repository/aim/view/dataImporter.jsp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java index 9e3e90b3087..457a1dd5463 100644 --- a/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/WEB-INF/src/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -213,7 +213,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } } if (dataImporterForm.getColumnPairs().isEmpty() || !dataImporterForm.getColumnPairs().containsValue("{projectTitle}")) { - response.setHeader("errorMessage", "You must have atleast the {projectTitle} key in your config."); + response.setHeader("errorMessage", "You must have at least the {projectTitle} key in your config."); response.setStatus(400); return mapping.findForward("importData"); } diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index 6eb50a37579..2ba30d98cac 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -77,6 +77,7 @@ $('.existing-config').change(function() { var configName = $(this).val(); + if (configName!=='none'){ var formData = new FormData(); formData.append("configName", configName); @@ -117,6 +118,7 @@ .catch(error => { console.error("There was a problem with the fetch operation:", error); }); + } }); }); From c0e50a9be10ac4a54d9fa0a1e62ea10e13d5b645 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Tue, 11 Jun 2024 09:09:53 +0300 Subject: [PATCH 029/229] AMP-30885 Mapping Ecowas file with importer(txt) --- amp/repository/aim/view/dataImporter.jsp | 39 +++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/amp/repository/aim/view/dataImporter.jsp b/amp/repository/aim/view/dataImporter.jsp index 2ba30d98cac..cf61b468b7c 100644 --- a/amp/repository/aim/view/dataImporter.jsp +++ b/amp/repository/aim/view/dataImporter.jsp @@ -285,15 +285,7 @@

Data Importer

- - + @@ -311,9 +303,7 @@ -
- - + + + + + + + + + +


-