Merge pull request #728 from Pixilib/new-export

Dissociate export study and export study-files api and make export database works as stream
Pixilib · Nov 27, 2024 · 3a12205 · 3a12205
2 parents e965076 + 0fb62c3
commit 3a12205
Show file tree

Hide file tree

Showing 23 changed files with 1,073 additions and 923 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - GaelO2
       - GaelO2-dev
+      - new-export
     tags:
       - '*'
 

diff --git a/GaelO2/app/GaelO/Adapters/ZipStreamAdapter.php b/GaelO2/app/GaelO/Adapters/ZipStreamAdapter.php
@@ -0,0 +1,42 @@
+<?php
+
+namespace App\GaelO\Adapters;
+
+use App\GaelO\Interfaces\Adapters\ZipStreamInterface;
+use Psr\Http\Message\StreamInterface;
+use ZipStream;
+
+class ZipStreamAdapter implements ZipStreamInterface
+{
+
+    private ZipStream\ZipStream $zipStream;
+
+    public function init(string $filename): void
+    {
+        $this->zipStream = new ZipStream\ZipStream(
+            outputName: $filename,
+            sendHttpHeaders: true,
+        );
+    }
+
+    public function addFileFromString(string $filename, string $content): void
+    {
+        $this->zipStream->addFile(
+            fileName: $filename,
+            data: $content,
+        );
+    }
+
+    public function addFileFromStream(string $filename, $stream): void
+    {
+        $this->zipStream->addFileFromStream(
+            fileName: $filename,
+            stream: $stream,
+        );
+    }
+
+    public function finish()
+    {
+        return $this->zipStream->finish();
+    }
+}
diff --git a/GaelO2/app/GaelO/Interfaces/Adapters/ZipStreamInterface.php b/GaelO2/app/GaelO/Interfaces/Adapters/ZipStreamInterface.php
@@ -0,0 +1,12 @@
+<?php
+
+namespace App\GaelO\Interfaces\Adapters;
+
+use Psr\Http\Message\StreamInterface;
+
+Interface ZipStreamInterface {
+    public function init(string $filename): void;
+    public function addFileFromString(string $filename, string $content): void;
+    public function addFileFromStream(string $filename, $stream) :void;
+    public function finish();
+}
diff --git a/GaelO2/app/GaelO/Services/ExportStudyService.php b/GaelO2/app/GaelO/Services/ExportStudyService.php
@@ -216,7 +216,7 @@ public function exportInvestigatorForms(): void
         $this->groupReviewPerVisitType($investigatorForms, Constants::ROLE_INVESTIGATOR);
     }
 
-    public function exportAll() :void
+    public function exportAllTables() :void
     {
         $this->exportPatientTable();
         $this->exportVisitTable();
@@ -225,7 +225,6 @@ public function exportAll() :void
         $this->exportReviewerForms();
         $this->exportTrackerTable();
         $this->exportUsersOfStudy();
-        $this->exportAssociatedFiles();
     }
 
     private function groupReviewPerVisitType(array $reviewEntities, string $role): void
@@ -303,21 +302,6 @@ public function exportTrackerTable(): void
         $this->exportStudyResults->setTrackerReviewResults($exportTrackerResult);
     }
 
-    public function exportAssociatedFiles(): void
-    {
-        $zip = new ZipArchive();
-        $tempZip = tempnam(ini_get('upload_tmp_dir'), 'TMPZIP_' . $this->studyName . '_');
-        $zip->open($tempZip, ZipArchive::OVERWRITE);
-        //Add a file to create zip
-        $zip->addFromString('Readme', 'Folder Containing associated files to study');
-        //send stored file for this study
-        Util::addStoredFilesInZipAndClose($zip, $this->studyName);
-
-        $exporFileResult = new ExportFileResults();
-        $exporFileResult->addExportFile(ExportDataResults::EXPORT_TYPE_ZIP, $tempZip);
-        $this->exportStudyResults->setExportFileResults($exporFileResult);
-    }
-
     public function getExportStudyResult(): ExportStudyResults
     {
         return $this->exportStudyResults;

diff --git a/GaelO2/app/GaelO/UseCases/ExportDatabase/ExportDatabase.php b/GaelO2/app/GaelO/UseCases/ExportDatabase/ExportDatabase.php
@@ -2,13 +2,13 @@
 
 namespace App\GaelO\UseCases\ExportDatabase;
 
+use App\GaelO\Adapters\FrameworkAdapter;
+use App\GaelO\Adapters\ZipStreamAdapter;
 use App\GaelO\Exceptions\AbstractGaelOException;
 use App\GaelO\Exceptions\GaelOForbiddenException;
 use App\GaelO\Interfaces\Adapters\DatabaseDumperInterface;
 use App\GaelO\Services\AuthorizationService\AuthorizationUserService;
-use App\GaelO\Util;
 use Exception;
-use ZipArchive;
 
 class ExportDatabase
 {
@@ -29,25 +29,9 @@ public function execute(ExportDatabaseRequest $exportDatabaseRequest, ExportData
             $this->checkAuthorization($exportDatabaseRequest->currentUserId);
             //Operation might be long, set max execution time to 30 minutes
             set_time_limit(1800);
-            $zip = new ZipArchive();
-            $tempZip = tempnam(ini_get('upload_tmp_dir'), 'TMPZIPDB_');
-            $zip->open($tempZip, ZipArchive::OVERWRITE);
-
-            $filePathSql = tempnam(ini_get('upload_tmp_dir'), 'TMPDB_');
-            $this->databaseDumperInterface->createDatabaseDumpFile($filePathSql);
-
-            $date = Date('Ymd_His');
-            $zip->addFile($filePathSql, "export_database_$date.sql");
-
-            Util::addStoredFilesInZipAndClose($zip, null);
-
-            //Unlick after lock released by zip close
-            unlink($filePathSql);
 
             $exportDatabaseResponse->status = 200;
             $exportDatabaseResponse->statusText = 'OK';
-            $exportDatabaseResponse->zipFile = $tempZip;
-            $exportDatabaseResponse->fileName = "export_database_" . $date . ".zip";
         } catch (AbstractGaelOException $e) {
             $exportDatabaseResponse->status = $e->statusCode;
             $exportDatabaseResponse->statusText = $e->statusText;
@@ -63,4 +47,25 @@ private function checkAuthorization(int $userId): void
             throw new GaelOForbiddenException();
         }
     }
+
+    public function readExport()
+    {
+        $date = Date('Ymd_His');
+        $zipStream = new ZipStreamAdapter();
+        $zipStream->init("export_database_" . $date . ".zip");
+        //send stored file for this study
+        try {
+            $filePathSql = tempnam(ini_get('upload_tmp_dir'), 'TMPDB_');
+            $this->databaseDumperInterface->createDatabaseDumpFile($filePathSql);
+            $zipStream->addFileFromStream("export_database_$date.sql", fopen($filePathSql, 'rb'), );
+            $files = FrameworkAdapter::getStoredFiles();
+            foreach ($files as $file) {
+                $fileStream = FrameworkAdapter::getFile($file, true);
+                $zipStream->addFileFromStream($file, $fileStream);
+            }
+            $zipStream->finish();
+        } finally {
+            unlink($filePathSql);
+        }
+    }
 }
diff --git a/GaelO2/app/GaelO/UseCases/ExportDatabase/ExportDatabaseResponse.php b/GaelO2/app/GaelO/UseCases/ExportDatabase/ExportDatabaseResponse.php
@@ -6,6 +6,4 @@ class ExportDatabaseResponse
 {
     public int $status;
     public string $statusText;
-    public string $zipFile;
-    public string $fileName;
 }
diff --git a/GaelO2/app/GaelO/UseCases/ExportStudyData/ExportStudyData.php b/GaelO2/app/GaelO/UseCases/ExportStudyData/ExportStudyData.php
@@ -25,22 +25,15 @@ public function execute(ExportStudyDataRequest $exportStudyDataRequest, ExportSt
     {
 
         try {
-
-            #increase memory limit as php zip stores metadata file in memory until final generation of zip
-            ini_set("memory_limit", "512")
-
             $studyName = $exportStudyDataRequest->studyName;
 
             $this->checkAuthorization($exportStudyDataRequest->currentUserId, $studyName);
 
-            //Operation might be long, set max execution time to 30 minutes
-            set_time_limit(1800);
-
             //Make this task continues even the users leave to prevent removing created temporary files
             ignore_user_abort(true);
 
             $this->exportStudyService->setStudyName($studyName);
-            $this->exportStudyService->exportAll();
+            $this->exportStudyService->exportAllTables();
             $exportResults = $this->exportStudyService->getExportStudyResult();
 
             $exportStudyDataResponse->zipFile = $exportResults->getResultsAsZip();

diff --git a/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFiles.php b/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFiles.php
@@ -0,0 +1,62 @@
+<?php
+
+namespace App\GaelO\UseCases\ExportStudyFiles;
+
+use App\GaelO\Constants\Constants;
+use App\GaelO\Exceptions\AbstractGaelOException;
+use App\GaelO\Exceptions\GaelOForbiddenException;
+use App\GaelO\Services\AuthorizationService\AuthorizationStudyService;
+use App\GaelO\Util;
+use Exception;
+
+class ExportStudyFiles
+{
+
+    private AuthorizationStudyService $authorizationStudyService;
+    private string $studyName;
+
+    public function __construct(AuthorizationStudyService $authorizationStudyService)
+    {
+        $this->authorizationStudyService = $authorizationStudyService;
+    }
+
+    public function execute(ExportStudyFilesRequest $exportStudyFilesRequest, ExportStudyFilesResponse $exportStudyFilesResponse)
+    {
+
+        try {
+
+            #increase memory limit as php zip stores metadata file in memory until final generation of zip
+            $studyName = $exportStudyFilesRequest->studyName;
+
+            $this->checkAuthorization($exportStudyFilesRequest->currentUserId, $studyName);
+
+            $this->studyName = $studyName;
+
+            //Operation might be long, set max execution time to 30 minutes
+            set_time_limit(1800);
+
+            $exportStudyFilesResponse->status = 200;
+            $exportStudyFilesResponse->statusText = 'OK';
+            $exportStudyFilesResponse->fileName = "export_files_" . $studyName . ".zip";
+        } catch (AbstractGaelOException $e) {
+            $exportStudyFilesResponse->status = $e->statusCode;
+            $exportStudyFilesResponse->statusText = $e->statusText;
+        } catch (Exception $e) {
+            throw $e;
+        }
+    }
+
+    private function checkAuthorization(int $currentUserId, string $studyName)
+    {
+
+        $this->authorizationStudyService->setStudyName($studyName);
+        $this->authorizationStudyService->setUserId($currentUserId);
+        if (!$this->authorizationStudyService->isAllowedStudy(Constants::ROLE_SUPERVISOR)) {
+            throw new GaelOForbiddenException();
+        }
+    }
+
+    public function readExport(){
+        Util::exportAssociatedFiles($this->studyName);
+    }
+}
diff --git a/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFilesRequest.php b/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFilesRequest.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\GaelO\UseCases\ExportStudyFiles;
+
+class ExportStudyFilesRequest
+{
+    public int $currentUserId;
+    public string $studyName;
+}
diff --git a/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFilesResponse.php b/GaelO2/app/GaelO/UseCases/ExportStudyFiles/ExportStudyFilesResponse.php
@@ -0,0 +1,13 @@
+<?php
+
+namespace App\GaelO\UseCases\ExportStudyFiles;
+
+use Psr\Http\Message\StreamInterface;
+
+class ExportStudyFilesResponse
+{
+    public int $status;
+    public string $statusText;
+    public StreamInterface $stream;
+    public string $fileName;
+}
diff --git a/GaelO2/app/GaelO/Util.php b/GaelO2/app/GaelO/Util.php
@@ -3,11 +3,15 @@
 namespace App\GaelO;
 
 use App\GaelO\Adapters\FrameworkAdapter;
+use App\GaelO\Adapters\ZipStreamAdapter;
+use App\GaelO\Interfaces\Adapters\ZipStreamInterface;
 use Carbon\Carbon;
 use FilesystemIterator;
+use Illuminate\Support\Facades\Log;
 use RecursiveDirectoryIterator;
 use RecursiveIteratorIterator;
 use ReflectionClass;
+use Throwable;
 use ZipArchive;
 
 class Util
@@ -146,4 +150,23 @@ public static function isUrlSafeString(string $value): bool
     {
         return preg_match('/^[a-zA-Z0-9_-]*$/', $value);
     }
+
+    public static function exportAssociatedFiles(string $studyName): ZipStreamInterface
+    {
+        $zipStream = new ZipStreamAdapter();
+        $zipStream->init('export_'. $studyName .'_files.zip');
+        $zipStream->addFileFromString('README', 'Folder Containing associated files the ' . $studyName . ' study');
+        //send stored file for this study
+        try {
+            $files = FrameworkAdapter::getStoredFiles($studyName);
+            foreach ($files as $file) {
+                $fileStream = FrameworkAdapter::getFile($file, true);
+                $zipStream->addFileFromStream($file, $fileStream);
+            }
+            $zipStream->finish();
+        } catch(Throwable $t) {
+            Log::error('Error building ZIP Achive');
+        }
+        return $zipStream;
+    }
 }
diff --git a/GaelO2/app/Http/Controllers/ExportDBController.php b/GaelO2/app/Http/Controllers/ExportDBController.php
@@ -11,22 +11,22 @@
 
 class ExportDBController extends Controller
 {
-    public function exportDB(Request $request, ExportDatabase $exportDatabase, ExportDatabaseRequest $exportDatabaseRequest, ExportDatabaseResponse $exportDatabaseResponse) {
+    public function exportDB(Request $request, ExportDatabase $exportDatabase, ExportDatabaseRequest $exportDatabaseRequest, ExportDatabaseResponse $exportDatabaseResponse)
+    {
         $currentUser = Auth::user();
         $requestData = $request->all();
-        
+
         Util::fillObject($requestData, $exportDatabaseRequest);
         $exportDatabaseRequest->currentUserId = $currentUser['id'];
-        
+
         $exportDatabase->execute($exportDatabaseRequest, $exportDatabaseResponse);
-        if($exportDatabaseResponse->status === 200){
-            return response()->download($exportDatabaseResponse->zipFile, $exportDatabaseResponse->fileName,
-                                            array('Content-Type: application/zip','Content-Length: '. filesize($exportDatabaseResponse->zipFile)))
-                            ->deleteFileAfterSend(true);
-        }else{
+        if ($exportDatabaseResponse->status === 200) {
+            return response()->stream(function () use ($exportDatabase): void {
+                $exportDatabase->readExport();
+            });
+        } else {
             return response()->noContent()
-            ->setStatusCode($exportDatabaseResponse->status, $exportDatabaseResponse->statusText);
+                ->setStatusCode($exportDatabaseResponse->status, $exportDatabaseResponse->statusText);
         }
-
     }
 }
diff --git a/GaelO2/app/Http/Controllers/StudyController.php b/GaelO2/app/Http/Controllers/StudyController.php
@@ -11,6 +11,9 @@
 use App\GaelO\UseCases\ExportStudyData\ExportStudyData;
 use App\GaelO\UseCases\ExportStudyData\ExportStudyDataRequest;
 use App\GaelO\UseCases\ExportStudyData\ExportStudyDataResponse;
+use App\GaelO\UseCases\ExportStudyFiles\ExportStudyFiles;
+use App\GaelO\UseCases\ExportStudyFiles\ExportStudyFilesRequest;
+use App\GaelO\UseCases\ExportStudyFiles\ExportStudyFilesResponse;
 use App\GaelO\UseCases\GetCreatablePatients\GetCreatablePatients;
 use App\GaelO\UseCases\GetCreatablePatients\GetCreatablePatientsRequest;
 use App\GaelO\UseCases\GetCreatablePatients\GetCreatablePatientsResponse;
@@ -238,6 +241,22 @@ public function exportStudyData(ExportStudyData $exportStudyData, ExportStudyDat
         }
     }
 
+    public function exportStudyFiles(ExportStudyFiles $exportStudyFiles, ExportStudyFilesRequest $exportStudyFilesRequest, ExportStudyFilesResponse $exportStudyFilesResponse, string $studyName)
+    {
+        $currentUser = Auth::user();
+        $exportStudyFilesRequest->currentUserId = $currentUser['id'];
+        $exportStudyFilesRequest->studyName = $studyName;
+
+        $exportStudyFiles->execute($exportStudyFilesRequest, $exportStudyFilesResponse);
+
+        if ($exportStudyFilesResponse->status === 200) {
+            return response()->stream(function () use ($exportStudyFiles): void {$exportStudyFiles->readExport();});
+        } else {
+            return response()->noContent()
+                ->setStatusCode($exportStudyFilesResponse->status, $exportStudyFilesResponse->statusText);
+        }
+    }
+
 
     public function getReviewsFromVisitType(Request $request, GetReviewsFromVisitType $getReviewsFromVisitType, GetReviewsFromVisitTypeRequest $getReviewsFromVisitTypeRequest, GetReviewsFromVisitTypeResponse $getReviewsFromVisitTypeResponse, string $studyName)
     {
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ on: @@
         branches:
           - GaelO2
           - GaelO2-dev
+          - new-export
         tags:
           - '*'
@@ Expand Down @@