Skip to content
This repository was archived by the owner on Jul 7, 2020. It is now read-only.

init commit for consecutive fail job counts #255

Open
satang500 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

init commit for consecutive fail job counts #255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions hydra-api/src/main/java/com/addthis/hydra/job/JobTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,10 @@ public int getErrors() {
return errors;
}

public void setErrors(int errors) {
this.errors = errors;
}

public int incrementErrors() {
return ++errors;
}
Expand Down
21 changes: 18 additions & 3 deletions hydra-main/src/main/java/com/addthis/hydra/job/spawn/Spawn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2762,7 +2762,7 @@ protected void handleMessage(CoreMessage core) {
}
} else if (core instanceof StatusTaskEnd) {
StatusTaskEnd end = (StatusTaskEnd) core;
log.info("[task.end] :: {}/{} exit={}", end.getJobUuid(), end.getNodeID(), end.getExitCode());
log.info("[task.end] :: {} / {} exit={}", end.getJobUuid(), end.getNodeID(), end.getExitCode());
SpawnMetrics.tasksCompletedPerHour.mark();
try {
job = getJob(end.getJobUuid());
Expand All @@ -2774,7 +2774,9 @@ protected void handleMessage(CoreMessage core) {
if (task.isRunning()) {
taskQueuesByPriority.incrementHostAvailableSlots(end.getHostUuid());
}
handleStatusTaskEnd(job, task, end);
if(handleStatusTaskEnd(job, task, end) == 1) {
resetTaskErrors(job);
}
}
}
} catch (Exception ex) {
Expand All @@ -2785,6 +2787,14 @@ protected void handleMessage(CoreMessage core) {
}
}

void resetTaskErrors(Job job) {
List<JobTask> tasks = job.getCopyOfTasks();
for(JobTask task : tasks) {
task.setErrors(0);
log.info("Error counter for consecutive job fails has been reset to 0 for job id = {}, host id = {}, taskId = {}", task.getJobUUID(), task.getHostUUID(), task.getTaskID());
}
}

/**
* Get a replacement host for a new task
*
Expand Down Expand Up @@ -3020,7 +3030,8 @@ private void stopTask(String jobUUID, int taskID, boolean force, boolean onlyIfQ
* @param task The task to modify
* @param update The message
*/
private void handleStatusTaskEnd(Job job, JobTask task, StatusTaskEnd update) {
int handleStatusTaskEnd(Job job, JobTask task, StatusTaskEnd update) {
int flag = 0;
TaskExitState exitState = update.getExitState();
boolean wasStopped = (exitState != null) && exitState.getWasStopped();
task.setFileCount(update.getFileCount());
Expand All @@ -3043,8 +3054,12 @@ private void handleStatusTaskEnd(Job job, JobTask task, StatusTaskEnd update) {
}
if (job.isFinished() && (update.getRebalanceSource() == null)) {
finishJob(job, errored);
if(!errored) {
flag = 1;
}
}
queueJobTaskUpdateEvent(job);
return flag;
}

/**
Expand Down