<feature>[longjob]: Introduce suspend long job message

conf/serviceConfig/longjob.xml

@@ -25,6 +25,9 @@
     <message>
         <name>org.zstack.header.longjob.APIResumeLongJobMsg</name>
     </message>
+    <message>
+        <name>org.zstack.header.longjob.APISuspendLongJobMsg</name>
+    </message>
     <message>
         <name>org.zstack.header.longjob.APICleanLongJobMsg</name>
     </message>

header/src/main/java/org/zstack/header/longjob/APISuspendLongJobEvent.java

@@ -0,0 +1,34 @@
+package org.zstack.header.longjob;
+
+import org.zstack.header.message.APIEvent;
+import org.zstack.header.rest.RestResponse;
+
+@RestResponse(allTo = "inventory")
+public class APISuspendLongJobEvent extends APIEvent {
+    private LongJobInventory inventory;
+
+    public LongJobInventory getInventory() {
+        return inventory;
+    }
+
+    public void setInventory(LongJobInventory inventory) {
+        this.inventory = inventory;
+    }
+
+    public APISuspendLongJobEvent() {
+        super();
+    }
+
+    public APISuspendLongJobEvent(String apiId) {
+        super(apiId);
+    }
+
+    public static APISuspendLongJobEvent __example__() {
+        APISuspendLongJobEvent event = new APISuspendLongJobEvent();
+        LongJobInventory inv = new LongJobInventory();
+        inv.setUuid(uuid());
+        inv.setState(LongJobState.Suspended);
+        event.setInventory(inv);
+        return event;
+    }
+}

header/src/main/java/org/zstack/header/longjob/APISuspendLongJobEventDoc_zh_cn.groovy

@@ -0,0 +1,32 @@
+package org.zstack.header.longjob
+
+import org.zstack.header.longjob.LongJobInventory
+import org.zstack.header.errorcode.ErrorCode
+
+doc {
+
+	title "在这里输入结构的名称"
+
+	ref {
+		name "inventory"
+		path "org.zstack.header.longjob.APISuspendLongJobEvent.inventory"
+		desc "null"
+		type "LongJobInventory"
+		since "5.5.0"
+		clz LongJobInventory.class
+	}
+	field {
+		name "success"
+		desc ""
+		type "boolean"
+		since "5.5.0"
+	}
+	ref {
+		name "error"
+		path "org.zstack.header.longjob.APISuspendLongJobEvent.error"
+		desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false
+		type "ErrorCode"
+		since "5.5.0"
+		clz ErrorCode.class
+	}
+}

header/src/main/java/org/zstack/header/longjob/APISuspendLongJobMsg.java

@@ -0,0 +1,38 @@
+package org.zstack.header.longjob;
+
+import org.springframework.http.HttpMethod;
+import org.zstack.header.identity.Action;
+import org.zstack.header.message.APIMessage;
+import org.zstack.header.message.APIParam;
+import org.zstack.header.rest.RestRequest;
+
+@Action(category = LongJobConstants.ACTION_CATEGORY)
+@RestRequest(
+        path = "/longjobs/{uuid}/actions",
+        isAction = true,
+        method = HttpMethod.PUT,
+        responseClass = APISuspendLongJobEvent.class
+)
+public class APISuspendLongJobMsg extends APIMessage implements LongJobMessage {
+    @APIParam(resourceType = LongJobVO.class, checkAccount = true)
+    private String uuid;
+
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setUuid(String uuid) {
+        this.uuid = uuid;
+    }
+
+    public static APISuspendLongJobMsg __example__() {
+        APISuspendLongJobMsg msg = new APISuspendLongJobMsg();
+        msg.setUuid(uuid());
+        return msg;
+    }
+
+    @Override
+    public String getLongJobUuid() {
+        return uuid;
+    }
+}

header/src/main/java/org/zstack/header/longjob/APISuspendLongJobMsgDoc_zh_cn.groovy

@@ -0,0 +1,58 @@
+package org.zstack.header.longjob
+
+import org.zstack.header.longjob.APISuspendLongJobEvent
+
+doc {
+    title "SuspendLongJob"
+
+    category "longjob"
+
+    desc """在这里填写API描述"""
+
+    rest {
+        request {
+			url "PUT /v1/longjobs/{uuid}/actions"
+
+			header (Authorization: 'OAuth the-session-uuid')
+
+            clz APISuspendLongJobMsg.class
+
+            desc """"""
+
+			params {
+
+				column {
+					name "uuid"
+					enclosedIn "suspendLongJob"
+					desc "资源的UUID，唯一标示该资源"
+					location "url"
+					type "String"
+					optional false
+					since "5.5.0"
+				}
+				column {
+					name "systemTags"
+					enclosedIn ""
+					desc "系统标签"
+					location "body"
+					type "List"
+					optional true
+					since "5.5.0"
+				}
+				column {
+					name "userTags"
+					enclosedIn ""
+					desc "用户标签"
+					location "body"
+					type "List"
+					optional true
+					since "5.5.0"
+				}
+			}
+        }
+
+        response {
+            clz APISuspendLongJobEvent.class
+        }
+    }
+}

header/src/main/java/org/zstack/header/longjob/LongJob.java

@@ -10,6 +10,7 @@
 public interface LongJob {
     void start(LongJobVO job, ReturnValueCompletion<APIEvent> completion);
     default void cancel(LongJobVO job, ReturnValueCompletion<Boolean> completion) {}
+    default void suspend(LongJobVO job, ReturnValueCompletion<Boolean> completion) {}
     default void resume(LongJobVO job, ReturnValueCompletion<APIEvent> completion) {}
     default void clean(LongJobVO job, NoErrorCompletion completion) {}
     default Class getAuditType() {

longjob/src/main/java/org/zstack/longjob/LongJobFactory.java

@@ -11,6 +11,7 @@ public interface LongJobFactory {
     LongJob getLongJob(String jobName);
     TreeMap<String, String> getFullJobName();
     boolean supportCancel(String jobName);
+    boolean supportSuspend(String jobName);
     boolean supportResume(String jobName);
     boolean supportClean(String jobName);
 }

longjob/src/main/java/org/zstack/longjob/LongJobFactoryImpl.java

@@ -30,6 +30,7 @@ public class LongJobFactoryImpl implements LongJobFactory, Component {
     private TreeMap<String, String> fullJobName = new TreeMap<>();
 
     private Set<String> notSupportCancelJobType = new HashSet<>();
+    private Set<String> notSupportSuspendJobType = new HashSet<>();
     private Set<String> notSupportResumeJobType = new HashSet<>();
     private Set<String> notSupportCleanJobType = new HashSet<>();
 
@@ -80,6 +81,11 @@ public boolean supportCancel(String jobName) {
         return !notSupportCancelJobType.contains(jobName);
     }
 
+    @Override
+    public boolean supportSuspend(String jobName) {
+        return !notSupportSuspendJobType.contains(jobName);
+    }
+
     @Override
     public boolean supportResume(String jobName) {
         return !notSupportResumeJobType.contains(jobName);
@@ -96,6 +102,10 @@ private void checkBehaviorSupported(String jobName, LongJob job) {
                 notSupportCancelJobType.add(jobName);
             }
 
+            if (method.getName().equals("suspend") && method.isDefault()) {
+                notSupportSuspendJobType.add(jobName);
+            }
+
             if (method.getName().equals("resume") && method.isDefault()) {
                 notSupportResumeJobType.add(jobName);
             }

longjob/src/main/java/org/zstack/longjob/LongJobManagerImpl.java

@@ -127,6 +127,8 @@ private void handleApiMessage(APIMessage msg) {
             handle((APIRerunLongJobMsg) msg);
         } else if (msg instanceof APIResumeLongJobMsg) {
             handle((APIResumeLongJobMsg) msg);
+        } else if (msg instanceof APISuspendLongJobMsg) {
+            handle((APISuspendLongJobMsg) msg);
         } else if (msg instanceof APICleanLongJobMsg) {
             handle((APICleanLongJobMsg) msg);
         } else {
@@ -424,6 +426,80 @@ public String getName() {
         });
     }
 
+    private void handle(APISuspendLongJobMsg msg) {
+        thdf.chainSubmit(new ChainTask(msg) {
+            @Override
+            public String getSyncSignature() {
+                return "longjob-" + msg.getUuid();
+            }
+
+            @Override
+            public void run(SyncTaskChain chain) {
+                final APISuspendLongJobEvent evt = new APISuspendLongJobEvent(msg.getId());
+                suspendLongJob(msg.getUuid(), new ReturnValueCompletion<LongJobVO>(chain) {
+                    @Override
+                    public void success(LongJobVO vo) {
+                        evt.setInventory(LongJobInventory.valueOf(vo));
+                        bus.publish(evt);
+                        chain.next();
+                    }
+
+                    @Override
+                    public void fail(ErrorCode errorCode) {
+                        evt.setError(errorCode);
+                        bus.publish(evt);
+                        chain.next();
+                    }
+                });
+            }
+
+            @Override
+            public String getName() {
+                return String.format("suspend-longjob-%s", msg.getUuid());
+            }
+        });
+    }
+
+    private void suspendLongJob(String uuid, ReturnValueCompletion<LongJobVO> completion) {
+        Tuple t = Q.New(LongJobVO.class).eq(LongJobVO_.uuid, uuid).select(LongJobVO_.state, LongJobVO_.jobName).findTuple();
+        LongJobState currentState = t.get(0, LongJobState.class);
+        String jobName = t.get(1, String.class);
+
+        if (currentState == LongJobState.Suspended) {
+            LongJobVO vo = dbf.findByUuid(uuid, LongJobVO.class);
+            completion.success(vo);
+            return;
+        }
+
+        if (currentState != LongJobState.Running) {
+            completion.fail(err(ORG_ZSTACK_LONGJOB_10002, LongJobErrors.NOT_SUPPORTED, "can only suspend running jobs, current state: %s", currentState));
+            return;
+        }
+
+        if (!longJobFactory.supportSuspend(jobName)) {
+            completion.fail(err(ORG_ZSTACK_LONGJOB_10002, LongJobErrors.NOT_SUPPORTED, "job type %s does not support suspend", jobName));
+            return;
+        }
+
+        LongJobVO vo = dbf.findByUuid(uuid, LongJobVO.class);
+        LongJob job = longJobFactory.getLongJob(vo.getJobName());
+
+        job.suspend(vo, new ReturnValueCompletion<Boolean>(completion) {
+            @Override
+            public void success(Boolean suspended) {
+                LongJobVO updatedVo = changeState(uuid, LongJobStateEvent.suspend);
+                logger.info(String.format("longjob [uuid:%s, name:%s] has been suspended", vo.getUuid(), vo.getName()));
+                completion.success(updatedVo);
+            }
+
+            @Override
+            public void fail(ErrorCode errorCode) {
+                logger.error(String.format("failed to suspend longjob [uuid:%s, name:%s]: %s", vo.getUuid(), vo.getName(), errorCode));
+                completion.fail(errorCode);
+            }
+        });
+    }
+
     private void handle(ResumeLongJobMsg msg) {
         thdf.chainSubmit(new ChainTask(msg) {
             @Override

longjob/src/main/java/org/zstack/longjob/LongJobUtils.java

@@ -162,11 +162,33 @@ static LongJobStateEvent getEventOnError(ErrorCode errorCode) {
             return LongJobStateEvent.suspend;
         } else if (errorCode.isError(LongJobErrors.CANCELED)) {
             return LongJobStateEvent.canceled;
+        } else if (isRecoverableError(errorCode)) {
+            return LongJobStateEvent.suspend;
         } else {
             return LongJobStateEvent.fail;
         }
     }
 
+    /**
+     * Check if an error is marked as recoverable in its opaque field.
+     * Any business module can mark an error as recoverable by setting
+     * "longJobRecoverable" to true in the error code's opaque field.
+     * This allows the long job framework to automatically suspend instead of fail
+     * for recoverable errors, enabling automatic retry after service restart.
+     *
+     * @param errorCode the error code to check
+     * @return true if the error is marked as recoverable, false otherwise
+     */
+    private static boolean isRecoverableError(ErrorCode errorCode) {
+        // Check if error code has recoverable flag in opaque
+        // Any business module can set this flag to indicate the error is recoverable
+        Object recoverable = errorCode.getFromOpaque("longJobRecoverable");
+        if (recoverable instanceof Boolean && (Boolean) recoverable) {
+            return true;
+        }
+        return false;
+    }
+
     private static void setExecuteTimeIfNeed(LongJobVO job) {
         if (job.getExecuteTime() == null) {
             long time = (System.currentTimeMillis() - job.getCreateDate().getTime()) / 1000;