fix: add retry mechanism while deleting project or process

Author: emptyOVO

inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleConstants.java

@@ -22,13 +22,16 @@ public class DolphinScheduleConstants {
     // DS public constants
     public static final String DS_ID = "id";
     public static final String DS_CODE = "code";
+    public static final String DS_SUCCESS = "success";
     public static final String DS_TOKEN = "token";
     public static final String DS_PAGE_SIZE = "pageSize";
     public static final String DS_PAGE_NO = "pageNo";
     public static final String DS_SEARCH_VAL = "searchVal";
     public static final String DS_RESPONSE_DATA = "data";
     public static final String DS_RESPONSE_NAME = "name";
     public static final String DS_RESPONSE_TOTAL_LIST = "totalList";
+    public static final int DS_DEFAULT_RETRY_TIMES = 3;
+    public static final int DS_DEFAULT_WAIT_MILLS = 1000;
     public static final String DS_DEFAULT_PAGE_SIZE = "10";
     public static final String DS_DEFAULT_PAGE_NO = "1";
     public static final String DS_DEFAULT_TIMEZONE_ID = "Asia/Shanghai";

inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleUtils.java

@@ -57,11 +57,13 @@
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_CODE;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_PAGE_NO;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_PAGE_SIZE;
+import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_RETRY_TIMES;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_SCHEDULE_TIME_FORMAT;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_TASK_DESC;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_TASK_GEN_NUM;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_TASK_NAME;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_TIMEZONE_ID;
+import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_DEFAULT_WAIT_MILLS;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_ID;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_ONLINE_URL;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_PAGE_NO;
@@ -78,6 +80,7 @@
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_RESPONSE_TOTAL_LIST;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_SCHEDULE_DEF;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_SEARCH_VAL;
+import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_SUCCESS;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_TASK_DEFINITION;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_TASK_GEN_NUM;
 import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinScheduleConstants.DS_TASK_RELATION;
@@ -89,6 +92,7 @@
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.JSON_PARSE_ERROR;
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.NETWORK_ERROR;
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.PROCESS_DEFINITION_CREATION_FAILED;
+import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.PROCESS_DEFINITION_IN_USED_ERROR;
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.PROCESS_DEFINITION_QUERY_FAILED;
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.PROCESS_DEFINITION_RELEASE_FAILED;
 import static org.apache.inlong.manager.schedule.exception.DolphinScheduleException.PROJECT_CREATION_FAILED;
@@ -489,17 +493,35 @@ public static void delete(String url, String token, long code) {
             Map<String, String> header = buildHeader(token);
 
             String requestUrl = url + "/" + code;
+            for (int attempt = 1; attempt <= DS_DEFAULT_RETRY_TIMES; attempt++) {
+                JsonObject response = executeHttpRequest(requestUrl, DELETE, new HashMap<>(), header);
 
-            JsonObject response = executeHttpRequest(requestUrl, DELETE, new HashMap<>(), header);
-            LOGGER.info("delete process or project success, response data: {}", response);
+                if (response.get(DS_SUCCESS).getAsBoolean()) {
+                    LOGGER.info("Delete process or project success, response data: {}", response);
+                    return;
+                }
+
+                if (response.get(DS_CODE).getAsInt() == PROCESS_DEFINITION_IN_USED_ERROR) {
+                    LOGGER.warn("Attempt {} of {}, retrying after {} ms...", attempt, DS_DEFAULT_RETRY_TIMES,
+                            DS_DEFAULT_WAIT_MILLS);
+                    Thread.sleep(DS_DEFAULT_WAIT_MILLS);
+                }
+            }
 
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            LOGGER.error("Thread interrupted while retrying delete process or project: ", e);
+            throw new DolphinScheduleException(
+                    DELETION_FAILED,
+                    String.format("Thread interrupted while retrying delete for code: %d at URL: %s", code, url), e);
         } catch (JsonParseException e) {
             LOGGER.error("JsonParseException during deleting process or project", e);
             throw new DolphinScheduleException(
                     JSON_PARSE_ERROR,
                     String.format("Error deleting process or project with code: %d at URL: %s", code, url), e);
 
         } catch (DolphinScheduleException e) {
+            LOGGER.error("Error deleting process or project: ", e);
             throw new DolphinScheduleException(
                     DELETION_FAILED,
                     String.format("Error deleting process or project with code: %d at URL: %s", code, url), e);

inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/exception/DolphinScheduleException.java

@@ -42,6 +42,7 @@ public class DolphinScheduleException extends RuntimeException {
     public static final String GEN_TASK_CODE_FAILED = "GEN_TASK_CODE_FAILED";
 
     // Process-related error codes
+    public static final int PROCESS_DEFINITION_IN_USED_ERROR = 10168;
     public static final String PROCESS_DEFINITION_QUERY_FAILED = "PROCESS_DEFINITION_QUERY_FAILED";
     public static final String PROCESS_DEFINITION_CREATION_FAILED = "PROCESS_DEFINITION_CREATION_FAILED";
     public static final String PROCESS_DEFINITION_RELEASE_FAILED = "PROCESS_DEFINITION_RELEASE_FAILED";