diff --git a/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java b/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java index d780de3ce3bfe499ef5f4d1d29ea7d1e274d51eb..8fffd93d9769c63da32b127b38fe1d4c4b0fbe7a 100644 --- a/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java +++ b/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java @@ -26,11 +26,13 @@ public abstract class AbstractRetryableJob extends AbstractNoticeableJob { /** * 重试当前节点 * @param scriptHistory + * @param errorType */ protected void retryCurrentNode(ScriptHistory scriptHistory, String errorType) { notice(scriptHistory, errorType); boolean retryable = scriptHistory.getScheduleId() != null && - (scriptHistory.getScheduleHistoryMode() == null || Constant.HistoryMode.RETRY.equals(scriptHistory.getScheduleHistoryMode())); + (scriptHistory.getScheduleHistoryMode() == null || Constant.HistoryMode.RETRY.equals(scriptHistory.getScheduleHistoryMode())) && + !"UNKNOWN".equals(scriptHistory.getJobFinalStatus()); if (retryable) { ScheduleSnapshot scheduleSnapshot = scheduleSnapshotService.findById(scriptHistory.getScheduleSnapshotId()); ScheduleSnapshot.Topology.Node node = scheduleSnapshot.analyzeCurrentNode(scriptHistory.getScheduleTopNodeId()); diff --git a/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java b/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java index 9b54e802a8ae36bed0e5f8f83bf6dbc13dfbf993..455108342e95885e2656e1bf5310f2c7af9e2477 100644 --- a/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java +++ b/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java @@ -113,15 +113,9 @@ public class ScriptHistoryYarnStateRefreshJob extends AbstractRetryableJob imple scriptHistory.updateState(httpYarnApp.getState()); } scriptHistory.setJobId(httpYarnApp.getId()); - scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); scriptHistory.setJobUrl(httpYarnApp.getTrackingUrl()); - if ("FAILED".equals(httpYarnApp.getFinalStatus())) { - scriptHistory.setErrors(httpYarnApp.getDiagnostics()); - } + scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); scriptHistory.setStartTime(new Date(httpYarnApp.getStartedTime())); - if (httpYarnApp.getFinishedTime() != 0) { - scriptHistory.setFinishTime(new Date(httpYarnApp.getFinishedTime())); - } scriptHistoryService.save(scriptHistory); } @@ -135,10 +129,19 @@ public class ScriptHistoryYarnStateRefreshJob extends AbstractRetryableJob imple } else { scriptHistory.updateState(httpYarnApp.getState()); } + scriptHistory.setJobId(httpYarnApp.getId()); + scriptHistory.setJobUrl(httpYarnApp.getTrackingUrl()); scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); if ("FAILED".equals(httpYarnApp.getFinalStatus())) { - scriptHistory.setErrors(httpYarnApp.getDiagnostics()); + if (httpYarnApp.getDiagnostics() != null) { + if (httpYarnApp.getDiagnostics().length() > 61440) { + scriptHistory.setErrors(httpYarnApp.getDiagnostics().substring(0, 61440)); + } else { + scriptHistory.setErrors(httpYarnApp.getDiagnostics()); + } + } } + scriptHistory.setStartTime(new Date(httpYarnApp.getStartedTime())); scriptHistory.setFinishTime(new Date(httpYarnApp.getFinishedTime())); } else { scriptHistory.updateState(Constant.JobState.FAILED); diff --git a/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java b/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java index c25543570da76021ed5131f46ea5310664a4b3f2..b80b9c613c876ebd9aa0a618a18d1eb743a5f114 100644 --- a/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java +++ b/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java @@ -12,13 +12,13 @@ import com.meiyou.bigwhale.job.ScriptHistoryShellRunnerJob; import com.meiyou.bigwhale.service.auth.UserService; import com.meiyou.bigwhale.util.SchedulerUtils; import com.meiyou.bigwhale.util.WebHdfsUtils; -import com.meiyou.bigwhale.util.YarnApiUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -558,8 +558,9 @@ public class ScriptServiceImpl extends AbstractMysqlPagingAndSortingQueryService String queueOld = dbScript.getQueue(); String queueReq = req.getQueue(); if (!dbScript.getClusterId().equals(req.getClusterId()) || !queueOld.equals(queueReq)) { - Cluster clusterOld = clusterService.findById(dbScript.getClusterId()); - return YarnApiUtils.getActiveApp(clusterOld.getYarnUrl(), dbScript.getUser(), queueOld, dbScript.getApp(), 3) != null; + // 只判断最近的一个任务状态,非最近的任务如果还在运行jobFinalStatus将会被更新为UNKNOWN + ScriptHistory scriptHistory = scriptHistoryService.findOneByQuery("scriptId=" + dbScript.getId(), new Sort(Sort.Direction.DESC, "createTime", "id")); + return scriptHistory != null && scriptHistory.isRunning(); } return false; }