[PLAT-14459] Continuous backups for s3

Summary:
This diff adds support for creating a scheduled backup task that backs up YBA and uploads the result to s3. Currently prometheus and releases data are not included.

Add cleanup logic for s3 bucket

Test Plan: Create config pointing to s3 bucket, ensure backups are uploaded every 2 minutes

Reviewers: vkumar, jmak, sanketh

Reviewed By: vkumar

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D37640

managed/src/main/java/api/v2/controllers/ContinuousBackupApiControllerImp.java

@@ -4,7 +4,7 @@
 
 import api.v2.handlers.ContinuousBackupHandler;
 import api.v2.models.ContinuousBackup;
-import api.v2.models.ContinuousBackupCreateSpec;
+import api.v2.models.ContinuousBackupSpec;
 import api.v2.models.ContinuousRestoreSpec;
 import api.v2.models.YBATask;
 import com.google.inject.Inject;
@@ -16,7 +16,7 @@ public class ContinuousBackupApiControllerImp extends ContinuousBackupApiControl
 
  @Override
  public ContinuousBackup createContinuousBackup(
- Http.Request request, UUID cUUID, ContinuousBackupCreateSpec continuousBackupCreateSpec)
+ Http.Request request, UUID cUUID, ContinuousBackupSpec continuousBackupCreateSpec)
  throws Exception {
  return cbHandler.createContinuousBackup(request, cUUID, continuousBackupCreateSpec);
  }
@@ -29,10 +29,7 @@ public ContinuousBackup deleteContinuousBackup(Http.Request request, UUID cUUID,
 
  @Override
  public ContinuousBackup editContinuousBackup(
- Http.Request request,
- UUID cUUID,
- UUID bUUID,
- ContinuousBackupCreateSpec continuousBackupCreateSpec)
+ Http.Request request, UUID cUUID, UUID bUUID, ContinuousBackupSpec continuousBackupCreateSpec)
  throws Exception {
  return cbHandler.editContinuousBackup(request, cUUID, bUUID, continuousBackupCreateSpec);
  }

managed/src/main/java/api/v2/handlers/ContinuousBackupHandler.java

@@ -5,17 +5,19 @@
 
 import api.v2.mappers.ContinuousBackupMapper;
 import api.v2.models.ContinuousBackup;
-import api.v2.models.ContinuousBackupCreateSpec;
+import api.v2.models.ContinuousBackupSpec;
 import api.v2.models.ContinuousRestoreSpec;
 import api.v2.models.YBATask;
 import api.v2.utils.ApiControllerUtils;
 import com.google.inject.Inject;
+import com.yugabyte.yw.commissioner.tasks.CreateYbaBackup;
 import com.yugabyte.yw.commissioner.tasks.RestoreContinuousBackup;
 import com.yugabyte.yw.common.PlatformServiceException;
 import com.yugabyte.yw.models.ContinuousBackupConfig;
 import com.yugabyte.yw.models.Customer;
+import com.yugabyte.yw.models.Schedule;
+import com.yugabyte.yw.models.helpers.TaskType;
 import com.yugabyte.yw.models.helpers.TimeUnit;
-import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 import play.mvc.Http;
@@ -25,16 +27,34 @@ public class ContinuousBackupHandler extends ApiControllerUtils {
  @Inject private YbaBackupHandler ybaBackupHandler;
 
  public ContinuousBackup createContinuousBackup(
- Http.Request request, UUID cUUID, ContinuousBackupCreateSpec continuousBackupCreateSpec)
+ Http.Request request, UUID cUUID, ContinuousBackupSpec continuousBackupCreateSpec)
  throws Exception {
 
+ // Check if there is an existing config
+ if (ContinuousBackupConfig.get().isPresent()) {
+ throw new PlatformServiceException(BAD_REQUEST, "Continuous backup config already exists.");
+ }
  ContinuousBackupConfig cbConfig =
  ContinuousBackupConfig.create(
  continuousBackupCreateSpec.getStorageConfigUuid(),
  continuousBackupCreateSpec.getFrequency(),
  TimeUnit.valueOf(continuousBackupCreateSpec.getFrequencyTimeUnit().name()),
  continuousBackupCreateSpec.getNumBackups(),
  continuousBackupCreateSpec.getBackupDir());
+ CreateYbaBackup.Params taskParams = new CreateYbaBackup.Params();
+ taskParams.storageConfigUUID = cbConfig.getStorageConfigUUID();
+ taskParams.dirName = cbConfig.getBackupDir();
+ // TODO: list of components?
+ Schedule schedule =
+ Schedule.create(
+ cUUID,
+ cbConfig.getUuid(),
+ taskParams,
+ TaskType.CreateYbaBackup,
+ cbConfig.getFrequency(),
+ null,
+ cbConfig.getFrequencyTimeUnit(),
+ null);
  return ContinuousBackupMapper.INSTANCE.toContinuousBackup(cbConfig);
  }
 
@@ -49,26 +69,24 @@ public ContinuousBackup deleteContinuousBackup(Http.Request request, UUID cUUID,
  }
 
  public ContinuousBackup editContinuousBackup(
- Http.Request request,
- UUID cUUID,
- UUID bUUID,
- ContinuousBackupCreateSpec continuousBackupCreateSpec)
+ Http.Request request, UUID cUUID, UUID bUUID, ContinuousBackupSpec continuousBackupCreateSpec)
  throws Exception {
  Optional<ContinuousBackupConfig> optional = ContinuousBackupConfig.get(bUUID);
  if (!optional.isPresent()) {
- throw new PlatformServiceException(BAD_REQUEST, "no continous backup config found with UUID");
+ throw new PlatformServiceException(
+ BAD_REQUEST, "No continous backup config found with UUID " + bUUID);
  }
  ContinuousBackupConfig cbConfig = optional.get();
  // TODO: Actual edit work
  return ContinuousBackupMapper.INSTANCE.toContinuousBackup(cbConfig);
  }
 
  public ContinuousBackup getContinuousBackup(Http.Request request, UUID cUUID) throws Exception {
- List<ContinuousBackupConfig> cbConfigs = ContinuousBackupConfig.getAll();
- if (cbConfigs.size() < 1) {
+ Optional<ContinuousBackupConfig> cbConfigOpt = ContinuousBackupConfig.get();
+ if (!cbConfigOpt.isPresent()) {
  throw new PlatformServiceException(NOT_FOUND, "No continuous backup config found.");
  }
- ContinuousBackupConfig cbConfig = cbConfigs.get(0);
+ ContinuousBackupConfig cbConfig = cbConfigOpt.get();
  return ContinuousBackupMapper.INSTANCE.toContinuousBackup(cbConfig);
  }
 

managed/src/main/java/api/v2/handlers/IsolatedBackupHandler.java

@@ -19,7 +19,7 @@ public YBATask createYbaBackup(Http.Request request, UUID cUUID, IsolatedBackupC
  throws Exception {
  Customer customer = Customer.getOrBadRequest(cUUID);
  CreateYbaBackup.Params taskParams = new CreateYbaBackup.Params();
- taskParams.localDir = spec.getLocalDir();
+ taskParams.dirName = spec.getLocalDir();
  taskParams.components = spec.getComponents();
  UUID taskUUID = ybaBackupHandler.createBackup(customer, taskParams);
  return new YBATask().taskUuid(taskUUID);

managed/src/main/java/api/v2/mappers/ContinuousBackupMapper.java

@@ -2,6 +2,7 @@
 
 import api.v2.models.ContinuousBackup;
 import api.v2.models.ContinuousBackupInfo;
+import api.v2.models.ContinuousBackupSpec;
 import api.v2.models.TimeUnitType;
 import com.yugabyte.yw.models.ContinuousBackupConfig;
 import java.time.OffsetDateTime;
@@ -15,14 +16,17 @@ public interface ContinuousBackupMapper {
  default ContinuousBackup toContinuousBackup(ContinuousBackupConfig cbConfig) {
  ContinuousBackup v2ContinuousBackup = new ContinuousBackup();
  ContinuousBackupInfo v2ContinuousBackupInfo = new ContinuousBackupInfo();
+ ContinuousBackupSpec v2ContinuousBackupSpec = new ContinuousBackupSpec();
  v2ContinuousBackupInfo.setUuid(cbConfig.getUuid());
- v2ContinuousBackupInfo.setFrequency(cbConfig.getFrequency());
- v2ContinuousBackupInfo.setFrequencyTimeUnit(
+ v2ContinuousBackupSpec.setStorageConfigUuid(cbConfig.getStorageConfigUUID());
+ v2ContinuousBackupSpec.setFrequency(cbConfig.getFrequency());
+ v2ContinuousBackupSpec.setFrequencyTimeUnit(
  TimeUnitType.valueOf(cbConfig.getFrequencyTimeUnit().name()));
  // TODO: compute from actual cbConfig
  v2ContinuousBackupInfo.setStorageLocation("s3://backup_bucket/YBA.1.2.3.4/");
  v2ContinuousBackupInfo.setLastBackup(OffsetDateTime.parse("2024-08-19T10:30:45-04:00"));
  v2ContinuousBackup.setInfo(v2ContinuousBackupInfo);
+ v2ContinuousBackup.setSpec(v2ContinuousBackupSpec);
  return v2ContinuousBackup;
  }
 }

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateYbaBackup.java

@@ -10,24 +10,56 @@
 
 package com.yugabyte.yw.commissioner.tasks;
 
+import static com.yugabyte.yw.common.Util.NULL_UUID;
+import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR;
+
 import api.v2.models.YbaComponent;
 import com.google.inject.Inject;
 import com.yugabyte.yw.commissioner.AbstractTaskBase;
 import com.yugabyte.yw.commissioner.BaseTaskDependencies;
+import com.yugabyte.yw.commissioner.Commissioner;
+import com.yugabyte.yw.common.CloudUtil;
+import com.yugabyte.yw.common.CloudUtilFactory;
+import com.yugabyte.yw.common.PlatformServiceException;
+import com.yugabyte.yw.common.ShellResponse;
+import com.yugabyte.yw.common.ha.PlatformReplicationHelper;
+import com.yugabyte.yw.common.ha.PlatformReplicationManager;
 import com.yugabyte.yw.forms.AbstractTaskParams;
+import com.yugabyte.yw.models.Customer;
+import com.yugabyte.yw.models.CustomerTask;
+import com.yugabyte.yw.models.Schedule;
+import com.yugabyte.yw.models.ScheduleTask;
+import com.yugabyte.yw.models.configs.CustomerConfig;
+import com.yugabyte.yw.models.helpers.TaskType;
+import java.io.File;
 import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
+import play.libs.Json;
 
 @Slf4j
 public class CreateYbaBackup extends AbstractTaskBase {
 
+ private final CloudUtilFactory cloudUtilFactory;
+ private final PlatformReplicationHelper replicationHelper;
+ private final PlatformReplicationManager replicationManager;
+
  @Inject
- protected CreateYbaBackup(BaseTaskDependencies baseTaskDependencies) {
+ protected CreateYbaBackup(
+ BaseTaskDependencies baseTaskDependencies,
+ PlatformReplicationHelper replicationHelper,
+ PlatformReplicationManager replicationManager,
+ CloudUtilFactory cloudUtilFactory) {
  super(baseTaskDependencies);
+ this.replicationHelper = replicationHelper;
+ this.replicationManager = replicationManager;
+ this.cloudUtilFactory = cloudUtilFactory;
  }
 
  public static class Params extends AbstractTaskParams {
- public String localDir;
+ public UUID storageConfigUUID;
+ public String dirName;
  public List<YbaComponent> components;
  }
 
@@ -36,9 +68,76 @@ protected Params taskParams() {
  return (Params) taskParams;
  }
 
+ public void runScheduledBackup(
+ Schedule schedule, Commissioner commissioner, boolean alreadyRunning) {
+ log.info("Execution of scheduled YBA backup");
+ if (alreadyRunning) {
+ log.info("Continuous backup already running, skipping.");
+ return;
+ }
+ UUID customerUUID = schedule.getCustomerUUID();
+ Customer customer = Customer.get(customerUUID);
+ CreateYbaBackup.Params taskParams =
+ Json.fromJson(schedule.getTaskParams(), CreateYbaBackup.Params.class);
+
+ if (schedule.isBacklogStatus()) {
+ schedule.updateBacklogStatus(false);
+ }
+
+ UUID taskUUID = commissioner.submit(TaskType.CreateYbaBackup, taskParams);
+ ScheduleTask.create(taskUUID, schedule.getScheduleUUID());
+ CustomerTask.create(
+ customer,
+ NULL_UUID,
+ taskUUID,
+ CustomerTask.TargetType.Yba,
+ CustomerTask.TaskType.CreateYbaBackup,
+ // TODO: Actually get platform IP
+ "platform_ip");
+ log.info("Submitted continuous yba backup creation with task uuid = {}.", taskUUID);
+ }
+
  @Override
  public void run() {
- log.info("Dummy exeuction of CreateYbaBackup");
+ log.info("Execution of CreateYbaBackup");
+ CreateYbaBackup.Params taskParams = taskParams();
+ if (taskParams.storageConfigUUID != null) {
+ log.debug("Creating platform backup...");
+ ShellResponse response =
+ replicationHelper.runCommand(replicationManager.new CreatePlatformBackupParams());
+
+ if (response.code != 0) {
+ log.error("Backup failed: " + response.message);
+ throw new PlatformServiceException(
+ INTERNAL_SERVER_ERROR, "Backup failed: " + response.message);
+ }
+ Optional<File> backupOpt = replicationHelper.getMostRecentBackup();
+ if (!backupOpt.isPresent()) {
+ throw new PlatformServiceException(INTERNAL_SERVER_ERROR, "could not find backup file");
+ }
+ File backup = backupOpt.get();
+ CustomerConfig customerConfig = CustomerConfig.get(taskParams.storageConfigUUID);
+ if (customerConfig == null) {
+ throw new PlatformServiceException(
+ INTERNAL_SERVER_ERROR,
+ "Could not find customer config with provided storage config UUID.");
+ }
+ CloudUtil cloudUtil = cloudUtilFactory.getCloudUtil(customerConfig.getName());
+ if (!cloudUtil.uploadYbaBackup(customerConfig.getDataObject(), backup, taskParams.dirName)) {
+ throw new PlatformServiceException(
+ INTERNAL_SERVER_ERROR, "Could not upload YBA backup to cloud storage.");
+ }
+
+ if (!cloudUtil.cleanupUploadedBackups(customerConfig.getDataObject(), taskParams.dirName)) {
+ log.warn(
+ "Error cleaning up uploaded backups to cloud storage, please delete manually to avoid"
+ + " incurring unexpected costs.");
+ }
+
+ // Cleanup backups
+ replicationHelper.cleanupCreatedBackups();
+ log.info(backup.getAbsolutePath());
+ }
  return;
  }
 }