From 1e24877c0dbba8456afd8cf05916021d5be1521f Mon Sep 17 00:00:00 2001 From: progr1mmer Date: Wed, 10 Mar 2021 15:37:24 +0800 Subject: [PATCH] =?UTF-8?q?Yarn=E7=8A=B6=E6=80=81=E6=9B=B4=E6=96=B0BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bigwhale/job/AbstractRetryableJob.java | 4 +++- .../job/ScriptHistoryYarnStateRefreshJob.java | 19 +++++++++++-------- .../bigwhale/service/ScriptServiceImpl.java | 7 ++++--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java b/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java index d780de3..8fffd93 100644 --- a/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java +++ b/src/main/java/com/meiyou/bigwhale/job/AbstractRetryableJob.java @@ -26,11 +26,13 @@ public abstract class AbstractRetryableJob extends AbstractNoticeableJob { /** * 重试当前节点 * @param scriptHistory + * @param errorType */ protected void retryCurrentNode(ScriptHistory scriptHistory, String errorType) { notice(scriptHistory, errorType); boolean retryable = scriptHistory.getScheduleId() != null && - (scriptHistory.getScheduleHistoryMode() == null || Constant.HistoryMode.RETRY.equals(scriptHistory.getScheduleHistoryMode())); + (scriptHistory.getScheduleHistoryMode() == null || Constant.HistoryMode.RETRY.equals(scriptHistory.getScheduleHistoryMode())) && + !"UNKNOWN".equals(scriptHistory.getJobFinalStatus()); if (retryable) { ScheduleSnapshot scheduleSnapshot = scheduleSnapshotService.findById(scriptHistory.getScheduleSnapshotId()); ScheduleSnapshot.Topology.Node node = scheduleSnapshot.analyzeCurrentNode(scriptHistory.getScheduleTopNodeId()); diff --git a/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java b/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java index 9b54e80..4551083 100644 --- a/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java +++ b/src/main/java/com/meiyou/bigwhale/job/ScriptHistoryYarnStateRefreshJob.java @@ -113,15 +113,9 @@ public class ScriptHistoryYarnStateRefreshJob extends AbstractRetryableJob imple scriptHistory.updateState(httpYarnApp.getState()); } scriptHistory.setJobId(httpYarnApp.getId()); - scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); scriptHistory.setJobUrl(httpYarnApp.getTrackingUrl()); - if ("FAILED".equals(httpYarnApp.getFinalStatus())) { - scriptHistory.setErrors(httpYarnApp.getDiagnostics()); - } + scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); scriptHistory.setStartTime(new Date(httpYarnApp.getStartedTime())); - if (httpYarnApp.getFinishedTime() != 0) { - scriptHistory.setFinishTime(new Date(httpYarnApp.getFinishedTime())); - } scriptHistoryService.save(scriptHistory); } @@ -135,10 +129,19 @@ public class ScriptHistoryYarnStateRefreshJob extends AbstractRetryableJob imple } else { scriptHistory.updateState(httpYarnApp.getState()); } + scriptHistory.setJobId(httpYarnApp.getId()); + scriptHistory.setJobUrl(httpYarnApp.getTrackingUrl()); scriptHistory.setJobFinalStatus(httpYarnApp.getFinalStatus()); if ("FAILED".equals(httpYarnApp.getFinalStatus())) { - scriptHistory.setErrors(httpYarnApp.getDiagnostics()); + if (httpYarnApp.getDiagnostics() != null) { + if (httpYarnApp.getDiagnostics().length() > 61440) { + scriptHistory.setErrors(httpYarnApp.getDiagnostics().substring(0, 61440)); + } else { + scriptHistory.setErrors(httpYarnApp.getDiagnostics()); + } + } } + scriptHistory.setStartTime(new Date(httpYarnApp.getStartedTime())); scriptHistory.setFinishTime(new Date(httpYarnApp.getFinishedTime())); } else { scriptHistory.updateState(Constant.JobState.FAILED); diff --git a/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java b/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java index c255435..b80b9c6 100644 --- a/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java +++ b/src/main/java/com/meiyou/bigwhale/service/ScriptServiceImpl.java @@ -12,13 +12,13 @@ import com.meiyou.bigwhale.job.ScriptHistoryShellRunnerJob; import com.meiyou.bigwhale.service.auth.UserService; import com.meiyou.bigwhale.util.SchedulerUtils; import com.meiyou.bigwhale.util.WebHdfsUtils; -import com.meiyou.bigwhale.util.YarnApiUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -558,8 +558,9 @@ public class ScriptServiceImpl extends AbstractMysqlPagingAndSortingQueryService String queueOld = dbScript.getQueue(); String queueReq = req.getQueue(); if (!dbScript.getClusterId().equals(req.getClusterId()) || !queueOld.equals(queueReq)) { - Cluster clusterOld = clusterService.findById(dbScript.getClusterId()); - return YarnApiUtils.getActiveApp(clusterOld.getYarnUrl(), dbScript.getUser(), queueOld, dbScript.getApp(), 3) != null; + // 只判断最近的一个任务状态，非最近的任务如果还在运行jobFinalStatus将会被更新为UNKNOWN + ScriptHistory scriptHistory = scriptHistoryService.findOneByQuery("scriptId=" + dbScript.getId(), new Sort(Sort.Direction.DESC, "createTime", "id")); + return scriptHistory != null && scriptHistory.isRunning(); } return false; } -- Gitee