From ff1f73db6e126ead29276852b4bf20fd04fb9065 Mon Sep 17 00:00:00 2001 From: Mihai Postelnicu Date: Thu, 8 Jun 2017 20:19:44 +0300 Subject: [PATCH 1/4] OCVN-400 Export all button not working on prod --- .../ocds/web/rest/controller/export/ExcelExportController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java index 714fb65b0..1bf285889 100644 --- a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java +++ b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java @@ -104,6 +104,7 @@ public void excelExport(@ModelAttribute @Valid final YearFilterPagingRequest fil response.setContentType("application/zip"); response.setHeader("Content-Disposition", "attachment; filename=" + "excel-export.zip"); + response.setContentLength((int) file.length()); InputStream is = new FileInputStream(file); IOUtils.copy(is, response.getOutputStream()); From 5a1cd2260609a73bfe28f9e7221453624dfabe3a Mon Sep 17 00:00:00 2001 From: Alexei Savca Date: Thu, 8 Jun 2017 22:22:01 +0300 Subject: [PATCH 2/4] OCVN-400 Had to enforce transfer-encoding: chunked --- .../rest/controller/export/ExcelExportController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java index 1bf285889..d66bf0fec 100644 --- a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java +++ b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java @@ -77,6 +77,10 @@ public void excelExport(@ModelAttribute @Valid final YearFilterPagingRequest fil response.getOutputStream().write(excelGenerator.getExcelDownload(filter)); } else { + response.setContentType("application/zip"); + response.setHeader("Content-Disposition", "attachment; filename=" + "excel-export.zip"); + response.flushBuffer(); + File file = File.createTempFile("createZip", ".zip"); logger.info("Created temp file: " + file.getAbsolutePath()); @@ -101,11 +105,6 @@ public void excelExport(@ModelAttribute @Valid final YearFilterPagingRequest fil zout.close(); fileCleaningTracker.track(file, file); IOUtils.closeQuietly(bos); - - response.setContentType("application/zip"); - response.setHeader("Content-Disposition", "attachment; filename=" + "excel-export.zip"); - response.setContentLength((int) file.length()); - InputStream is = new FileInputStream(file); IOUtils.copy(is, response.getOutputStream()); response.flushBuffer(); From f1789e57b82f32c665be86ff5b2e33d9ed16a346 Mon Sep 17 00:00:00 2001 From: Alexei Savca Date: Thu, 8 Jun 2017 22:36:14 +0300 Subject: [PATCH 3/4] OCVN-400 Now giving a direct link for excel download --- ui/oce/index.jsx | 24 ++++++++++++------------ ui/oce/style.less | 5 +++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ui/oce/index.jsx b/ui/oce/index.jsx index 2d0c50a5a..02893a1a3 100644 --- a/ui/oce/index.jsx +++ b/ui/oce/index.jsx @@ -298,22 +298,22 @@ class OCApp extends React.Component{ } exportBtn(){ - if(this.state.exporting){ - return ( -
-
-
- {this.t('export:exporting')} -
-
-
- ) + const { filters, selectedYears, locale, selectedMonths } = this.state; + let url = new URI('/api/ocds/excelExport') + .addSearch(filters.toJS()) + .addSearch('year', selectedYears.toArray()) + .addSearch('language', locale); + + if(selectedYears.count() == 1){ + url = url.addSearch('month', selectedMonths && selectedMonths.toJS()) + .addSearch('monthly', true); } - return
this.downloadExcel()}> + + return {this.t('export:export')} -
+ } toggleDashboardSwitcher(e){ diff --git a/ui/oce/style.less b/ui/oce/style.less index 68d3a2f4f..eff6f84e0 100644 --- a/ui/oce/style.less +++ b/ui/oce/style.less @@ -63,3 +63,8 @@ } } } + +a.filters{ + text-decoration: none; + color: white; +} From 8671b92f604eb85ae59b2a14bcdf79096110c471 Mon Sep 17 00:00:00 2001 From: Mihai Postelnicu Date: Thu, 8 Jun 2017 23:52:55 +0300 Subject: [PATCH 4/4] OCE-400 improved file export, download progress will be properly reported, no temp files --- .../export/ExcelExportController.java | 47 +++++-------------- .../ocds/web/util/SettingsUtils.java | 5 +- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java index d66bf0fec..81a85b923 100644 --- a/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java +++ b/web/src/main/java/org/devgateway/ocds/web/rest/controller/export/ExcelExportController.java @@ -1,19 +1,7 @@ package org.devgateway.ocds.web.rest.controller.export; import io.swagger.annotations.ApiOperation; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.Deflater; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import org.apache.commons.io.FileCleaningTracker; -import org.apache.commons.io.IOUtils; import org.devgateway.ocds.persistence.mongo.Release; import org.devgateway.ocds.persistence.mongo.constants.MongoConstants; import org.devgateway.ocds.web.rest.controller.GenericOCDSController; @@ -29,6 +17,13 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import static org.springframework.data.mongodb.core.query.Query.query; @@ -39,19 +34,15 @@ @RestController public class ExcelExportController extends GenericOCDSController { protected final Logger logger = LoggerFactory.getLogger(ExcelExportController.class); - + @Autowired + protected AdminSettingsRepository adminSettingsRepository; @Autowired private SettingsUtils settingsUtils; - @Autowired private ExcelGenerator excelGenerator; - @Autowired private FileCleaningTracker fileCleaningTracker; - @Autowired - protected AdminSettingsRepository adminSettingsRepository; - @ApiOperation(value = "Export releases in Excel format.") @RequestMapping(value = "/api/ocds/excelExport", method = {RequestMethod.GET, RequestMethod.POST}) public void excelExport(@ModelAttribute @Valid final YearFilterPagingRequest filter, @@ -74,39 +65,25 @@ public void excelExport(@ModelAttribute @Valid final YearFilterPagingRequest fil if (numberOfReleases <= settingsUtils.getExcelBatchSize()) { response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); response.setHeader("Content-Disposition", "attachment; filename=" + "excel-export.xlsx"); - response.getOutputStream().write(excelGenerator.getExcelDownload(filter)); } else { response.setContentType("application/zip"); response.setHeader("Content-Disposition", "attachment; filename=" + "excel-export.zip"); response.flushBuffer(); - - File file = File.createTempFile("createZip", ".zip"); - logger.info("Created temp file: " + file.getAbsolutePath()); - - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file)); - - ZipOutputStream zout = new ZipOutputStream(bos); + ZipOutputStream zout = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream())); zout.setMethod(ZipOutputStream.DEFLATED); - zout.setLevel(Deflater.BEST_SPEED); - + zout.setLevel(Deflater.NO_COMPRESSION); int numberOfPages = (int) Math.ceil((double) numberOfReleases / filter.getPageSize()); for (int i = 0; i < numberOfPages; i++) { filter.setPageNumber(i); - ZipEntry ze = new ZipEntry("excel-export-page " + (i + 1) + ".xlsx"); - zout.putNextEntry(ze); byte[] bytes = excelGenerator.getExcelDownload(filter); zout.write(bytes, 0, bytes.length); - zout.closeEntry(); + response.flushBuffer(); } zout.close(); - fileCleaningTracker.track(file, file); - IOUtils.closeQuietly(bos); - InputStream is = new FileInputStream(file); - IOUtils.copy(is, response.getOutputStream()); response.flushBuffer(); } } diff --git a/web/src/main/java/org/devgateway/ocds/web/util/SettingsUtils.java b/web/src/main/java/org/devgateway/ocds/web/util/SettingsUtils.java index 88111202f..3f07f5d70 100644 --- a/web/src/main/java/org/devgateway/ocds/web/util/SettingsUtils.java +++ b/web/src/main/java/org/devgateway/ocds/web/util/SettingsUtils.java @@ -1,6 +1,5 @@ package org.devgateway.ocds.web.util; -import java.util.List; import org.devgateway.toolkit.persistence.dao.AdminSettings; import org.devgateway.toolkit.persistence.repository.AdminSettingsRepository; import org.slf4j.Logger; @@ -8,6 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.List; + /** * @author idobre * @since 6/22/16 @@ -18,7 +19,7 @@ public class SettingsUtils { public static final String DEFAULT_LANGUAGE = "en_US"; - private static final Integer EXCELBATCHSIZEDEFAULT = 10000; + private static final Integer EXCELBATCHSIZEDEFAULT = 5000; @Autowired private AdminSettingsRepository adminSettingsRepository;