HBASE-29240: Backup ancestry trees should have the ability to be invalidated

Author: Hernan Gelaf-Romer

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupClientFactory.java

@@ -18,15 +18,24 @@
 package org.apache.hadoop.hbase.backup;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
+import org.apache.hadoop.hbase.backup.impl.BackupManager;
 import org.apache.hadoop.hbase.backup.impl.FullTableBackupClient;
 import org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient;
+import org.apache.hadoop.hbase.backup.impl.InvalidAncestryException;
 import org.apache.hadoop.hbase.backup.impl.TableBackupClient;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 public final class BackupClientFactory {
+  private static final Logger LOG = LoggerFactory.getLogger(BackupClientFactory.class);
+
   private BackupClientFactory() {
   }
 
@@ -49,8 +58,42 @@ public static TableBackupClient create(Connection conn, String backupId, BackupR
     BackupType type = request.getBackupType();
     if (type == BackupType.FULL) {
       return new FullTableBackupClient(conn, backupId, request);
-    } else {
-      return new IncrementalTableBackupClient(conn, backupId, request);
     }
+
+    String latestFullBackup = getLatestFullBackupId(conn, request);
+
+    try (BackupAdmin admin = new BackupAdminImpl(conn)) {
+      boolean isInvalidAncestry = admin.getBackupInfo(latestFullBackup).isInvalidAncestry();
+
+      if (!isInvalidAncestry) {
+        return new IncrementalTableBackupClient(conn, backupId, request);
+      }
+
+      if (request.getFailOnInvalidAncestry()) {
+        throw new InvalidAncestryException(request);
+      }
+
+      LOG.info("Ancestry is invalid for backupId {}, creating a full backup", latestFullBackup);
+      return new FullTableBackupClient(conn, backupId, request);
+    }
+  }
+
+  private static String getLatestFullBackupId(Connection conn, BackupRequest request)
+    throws IOException {
+    try (BackupManager backupManager = new BackupManager(conn, conn.getConfiguration())) {
+      // Sorted in desc order by time
+      List<BackupInfo> backups = backupManager.getBackupHistory(true);
+
+      for (BackupInfo info : backups) {
+        if (
+          info.getType() == BackupType.FULL
+            && Objects.equals(info.getBackupRootDir(), request.getTargetRootDir())
+        ) {
+          return info.getBackupId();
+        }
+      }
+    }
+    throw new RuntimeException(
+      "Could not find a valid full backup for incremental request " + request);
   }
 }

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java

@@ -170,6 +170,8 @@ public enum BackupPhase {
    */
   private boolean noChecksumVerify;
 
+  private boolean isInvalidAncestry = false;
+
   public BackupInfo() {
     backupTableInfoMap = new HashMap<>();
   }
@@ -203,6 +205,14 @@ public void setBandwidth(long bandwidth) {
     this.bandwidth = bandwidth;
   }
 
+  public void setInvalidAncestry(boolean invalidAncestry) {
+    isInvalidAncestry = invalidAncestry;
+  }
+
+  public boolean isInvalidAncestry() {
+    return isInvalidAncestry;
+  }
+
   public void setNoChecksumVerify(boolean noChecksumVerify) {
     this.noChecksumVerify = noChecksumVerify;
   }
@@ -423,6 +433,7 @@ public BackupProtos.BackupInfo toProtosBackupInfo() {
     builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name()));
     builder.setWorkersNumber(workers);
     builder.setBandwidth(bandwidth);
+    builder.setIsInvalidAncestry(isInvalidAncestry);
     return builder.build();
   }
 
@@ -518,6 +529,7 @@ public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
     context.setType(BackupType.valueOf(proto.getBackupType().name()));
     context.setWorkers(proto.getWorkersNumber());
     context.setBandwidth(proto.getBandwidth());
+    context.setInvalidAncestry(proto.getIsInvalidAncestry());
     return context;
   }
 

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupMasterObserver.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class BackupMasterObserver implements MasterObserver, MasterCoprocessor {
+
+  private final Optional<MasterObserver> observer;
+
+  public BackupMasterObserver() {
+    this.observer = Optional.of(this);
+  }
+
+  @Override
+  public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+    TableName tableName) throws IOException {
+    BackupSystemTable table = new BackupSystemTable(ctx.getEnvironment().getConnection());
+    try {
+      table.startBackupExclusiveOperation();
+      table.invalidateTableAncestry(tableName);
+    } finally {
+      table.finishBackupExclusiveOperation();
+    }
+  }
+
+  @Override
+  public Optional<MasterObserver> getMasterObserver() {
+    return observer;
+  }
+}

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java

@@ -35,6 +35,11 @@ public Builder() {
       request = new BackupRequest();
     }
 
+    public Builder withFailOnInvalidAncestry(boolean failOnInvalidAncestry) {
+      request.failOnInvalidAncestry = failOnInvalidAncestry;
+      return this;
+    }
+
     public Builder withBackupType(BackupType type) {
       request.setBackupType(type);
       return this;
@@ -89,10 +94,20 @@ public BackupRequest build() {
   private boolean noChecksumVerify = false;
   private String backupSetName;
   private String yarnPoolName;
+  private boolean failOnInvalidAncestry = false;
 
   private BackupRequest() {
   }
 
+  private BackupRequest setFailOnInvalidAncestry(boolean failOnInvalidAncestry) {
+    this.failOnInvalidAncestry = failOnInvalidAncestry;
+    return this;
+  }
+
+  public boolean getFailOnInvalidAncestry() {
+    return failOnInvalidAncestry;
+  }
+
   private BackupRequest setBackupType(BackupType type) {
     this.type = type;
     return this;

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.backup.BackupHFileCleaner;
 import org.apache.hadoop.hbase.backup.BackupInfo;
 import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
+import org.apache.hadoop.hbase.backup.BackupMasterObserver;
 import org.apache.hadoop.hbase.backup.BackupObserver;
 import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
 import org.apache.hadoop.hbase.backup.BackupType;
@@ -125,6 +126,10 @@ public static void decorateMasterConfiguration(Configuration conf) {
           + "Added master procedure manager: {}",
         cleanerClass, masterProcedureClass, BackupHFileCleaner.class.getName());
     }
+
+    String observers = conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
+    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+      (observers == null ? "" : observers + ",") + BackupMasterObserver.class.getName());
   }
 
   /**

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java

@@ -277,6 +277,29 @@ public void close() {
     // do nothing
   }
 
+  public void invalidateTableAncestry(TableName toInvalidate) throws IOException {
+    List<BackupInfo> fullTableBackups = getCompletedFullBackups();
+    List<Put> invalidatePuts = new ArrayList<>(fullTableBackups.size());
+
+    for (BackupInfo backupInfo : fullTableBackups) {
+      // to minimize the amount of mutations against the backup system table, we only
+      // need to update full backups that have currently have a valid ancestry line
+      if (
+        backupInfo.getTables().contains(toInvalidate) && backupInfo.getType() == BackupType.FULL
+          && !backupInfo.isInvalidAncestry()
+      ) {
+        backupInfo.setInvalidAncestry(true);
+        invalidatePuts.add(createPutForBackupInfo(backupInfo));
+        LOG.info("Invalidating ancestry for backup {} due to table {}", backupInfo.getBackupId(),
+          toInvalidate);
+      }
+    }
+
+    try (BufferedMutator mutator = connection.getBufferedMutator(tableName)) {
+      mutator.mutate(invalidatePuts);
+    }
+  }
+
   /**
    * Updates status (state) of a backup session in backup system table table
    * @param info backup info
@@ -841,6 +864,24 @@ public ArrayList<BackupInfo> getBackupInfos(BackupState state) throws IOExceptio
     }
   }
 
+  private List<BackupInfo> getCompletedFullBackups() throws IOException {
+    Scan scan = createScanForBackupHistory();
+    List<BackupInfo> backups = new ArrayList<>();
+
+    try (Table table = connection.getTable(tableName)) {
+      ResultScanner scanner = table.getScanner(scan);
+      Result res;
+      while ((res = scanner.next()) != null) {
+        res.advance();
+        BackupInfo context = cellToBackupInfo(res.current());
+        if (context.getState() == BackupState.COMPLETE && context.getType() == BackupType.FULL) {
+          backups.add(context);
+        }
+      }
+    }
+    return backups;
+  }
+
   /**
    * Write the current timestamps for each regionserver to backup system table after a successful
    * full or incremental backup. The saved timestamp is of the last log file that was backed up

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/InvalidAncestryException.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.impl;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.backup.BackupRequest;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class InvalidAncestryException extends HBaseIOException {
+  public InvalidAncestryException(BackupRequest request) {
+    super("Could not take incremental backup %s because it has an invalid ancestry tree"
+      .formatted(request));
+  }
+}