Migrate project with google storage backend

keboola · Jan 16, 2025 · 721ea10 · 721ea10
1 parent 0134e97
commit 721ea10
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 17 deletions.
diff --git a/src/JobRunner/JobRunner.php b/src/JobRunner/JobRunner.php
@@ -23,9 +23,14 @@ public function __construct(Client $client, LoggerInterface $logger)
         $this->logger = $logger;
     }
 
-    abstract public function runJob(string $componentId, array $data): array;
-
-    abstract public function runSyncAction(string $componentId, string $action, array $data): array;
+    abstract public function runJob(string $componentId, array $data, ?string $tag = null): array;
+
+    abstract public function runSyncAction(
+        string $componentId,
+        string $action,
+        array $data,
+        ?string $tag = null
+    ): array;
 
     public function getServiceUrl(string $serviceId): string
     {

diff --git a/src/JobRunner/QueueV2JobRunner.php b/src/JobRunner/QueueV2JobRunner.php
@@ -13,9 +13,9 @@ class QueueV2JobRunner extends JobRunner
 {
     private const MAX_DELAY = 10;
 
-    public function runJob(string $componentId, array $data): array
+    public function runJob(string $componentId, array $data, ?string $tag = null): array
     {
-        $jobData = new JobData($componentId, null, $data);
+        $jobData = new JobData($componentId, null, $data, 'run', [], $tag);
         $response = $this->getQueueClient()->createJob($jobData);
 
         $attempt = 0;
@@ -30,11 +30,11 @@ public function runJob(string $componentId, array $data): array
         return $job;
     }
 
-    public function runSyncAction(string $componentId, string $action, array $data): array
+    public function runSyncAction(string $componentId, string $action, array $data, ?string $tag = null): array
     {
         $client = $this->getSyncActionsClient();
 
-        $data = new ActionData($componentId, $action, $data);
+        $data = new ActionData($componentId, $action, $data, $tag);
 
         return $client->callAction($data);
     }

diff --git a/src/JobRunner/SyrupJobRunner.php b/src/JobRunner/SyrupJobRunner.php
@@ -8,15 +8,19 @@
 
 class SyrupJobRunner extends JobRunner
 {
-    public function runJob(string $componentId, array $data): array
+    public function runJob(string $componentId, array $data, ?string $tag = null): array
     {
+        $options = ['configData' => $data];
+        if ($tag) {
+            $options['tag'] = $tag;
+        }
         return $this->getSyrupClient()->runJob(
             $componentId,
-            ['configData' => $data]
+            $options
         );
     }
 
-    public function runSyncAction(string $componentId, string $action, array $data): array
+    public function runSyncAction(string $componentId, string $action, array $data, ?string $tag = null): array
     {
         return $this->getSyrupClient(1)->runSyncAction(
             $this->getServiceUrl('docker-runner'),

diff --git a/src/Migrate.php b/src/Migrate.php
@@ -100,9 +100,11 @@ public function __construct(
 
     public function run(): void
     {
-        $restoreCredentials = $this->generateBackupCredentials();
         try {
-            $this->backupSourceProject($restoreCredentials['backupId']);
+            $backupId = $this->sourceProjectStorageClient->generateId();
+            $this->backupSourceProject($backupId);
+            $restoreCredentials = $this->generateBackupCredentials($backupId);
+
             $this->restoreDestinationProject($restoreCredentials);
 
             if ($this->migrateSecrets) {
@@ -129,7 +131,7 @@ public function run(): void
         }
     }
 
-    private function generateBackupCredentials(): array
+    private function generateBackupCredentials(string $backupId): array
     {
         $this->logger->info('Creating backup credentials');
 
@@ -138,9 +140,10 @@ private function generateBackupCredentials(): array
             'generate-read-credentials',
             [
                 'parameters' => [
-                    'backupId' => null,
+                    'backupId' => $backupId,
                 ],
-            ]
+            ],
+            'PST-2373-ondra'
         );
     }
 
@@ -155,7 +158,8 @@ private function backupSourceProject(string $backupId): void
                     'backupId' => $backupId,
                     'exportStructureOnly' => $this->directDataMigration || $this->migrateStructureOnly,
                 ],
-            ]
+            ],
+            'PST-2373-ondra'
         );
         if ($job['status'] !== self::JOB_STATUS_SUCCESS) {
             throw new UserException('Project snapshot create error: ' . $job['result']['message']);
@@ -172,7 +176,8 @@ private function restoreDestinationProject(array $restoreCredentials): void
 
         $job = $this->destJobRunner->runJob(
             Config::PROJECT_RESTORE_COMPONENT,
-            $configData
+            $configData,
+            'PST-2375-ondra'
         );
 
         if ($job['status'] !== self::JOB_STATUS_SUCCESS) {
@@ -344,6 +349,24 @@ private function getRestoreConfigData(array $restoreCredentials): array
                     'restorePermanentFiles' => $this->migratePermanentFiles,
                 ],
             ];
+        } elseif (isset($restoreCredentials['credentials']['accessToken'])) {
+            return [
+                'parameters' => [
+                    'gcs' => [
+                        'projectId' => $restoreCredentials['projectId'],
+                        'bucket' => $restoreCredentials['bucket'],
+                        'backupUri' => $restoreCredentials['backupUri'],
+                        'credentials' => [
+                            'accessToken' => $restoreCredentials['credentials']['accessToken'],
+                            'expiresIn' => $restoreCredentials['credentials']['expiresIn'],
+                            'tokenType' => $restoreCredentials['credentials']['tokenType'],
+                        ],
+                    ],
+                    'useDefaultBackend' => true,
+                    'restoreConfigs' => $this->migrateSecrets === false,
+                    'restorePermanentFiles' => $this->migratePermanentFiles,
+                ],
+            ];
         } else {
             throw new UserException('Unrecognized restore credentials.');
         }