@Override
public Label getAssignedLabel(SubTask task) {
return label;
}
/**
* {@link ComputerLauncher#afterDisconnect(SlaveComputer, TaskListener)}
* <p>
* EC2 Fleet plugin overrides this method to detect jobs which were failed because of
* EC2 instance was terminated/stopped. It could be manual stop or because of Spot marked.
* In all cases as soon as job aborted because of broken connection and slave is offline
* it will try to resubmit aborted job back to the queue, so user doesn't need to do that manually
* and another slave could take it.
* <p>
* Implementation details
* <p>
* There is no official recommendation about way how to resubmit job according to
* https://issues.jenkins-ci.org/browse/JENKINS-49707 moreover some of Jenkins code says it impossible.
* <p>
* method checks {@link SlaveComputer#getOfflineCause()} for disconnect because of EC2 instance termination
* it returns
* <code>
* result = {[email protected]} "Connection was broken: java.io.IOException:
* Unexpected termination of the channel\n\tat hudson.remoting.SynchronousCommandTransport$ReaderThread...
* cause = {[email protected]} "java.io.IOException: Unexpected termination of the channel"
* timestamp = 1561067177837
* </code>
*
* @param computer computer
* @param listener listener
*/
@Override
public void afterDisconnect(final SlaveComputer computer, final TaskListener listener) {
// according to jenkins docs could be null in edge cases, check ComputerLauncher.afterDisconnect
if (computer == null) return;
// in some multi-thread edge cases cloud could be null for some time, just be ok with that
final EC2FleetCloud cloud = ((EC2FleetNodeComputer) computer).getCloud();
if (cloud == null) {
LOGGER.warning("Edge case cloud is null for computer " + computer.getDisplayName()
+ " should be autofixed in a few minutes, if no please create issue for plugin");
return;
}
final boolean unexpectedDisconnect = computer.isOffline() && computer.getOfflineCause() instanceof OfflineCause.ChannelTermination;
if (!cloud.isDisableTaskResubmit() && unexpectedDisconnect) {
final List<Executor> executors = computer.getExecutors();
LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
+ " termination, resubmit");
for (Executor executor : executors) {
if (executor.getCurrentExecutable() != null) {
executor.interrupt(Result.ABORTED, new EC2TerminationCause(computer.getDisplayName()));
final Queue.Executable executable = executor.getCurrentExecutable();
// if executor is not idle
if (executable != null) {
final SubTask subTask = executable.getParent();
final Queue.Task task = subTask.getOwnerTask();
List<Action> actions = new ArrayList<>();
if (executable instanceof Actionable) {
actions = ((Actionable) executable).getActions();
}
Queue.getInstance().schedule2(task, RESCHEDULE_QUIET_PERIOD_SEC, actions);
LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
+ " termination, resubmit " + task + " with actions " + actions);
}
}
}
LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
+ " termination, resubmit finished");
} else {
LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
+ " termination but resubmit disabled, no actions, disableTaskResubmit: "
+ cloud.isDisableTaskResubmit() + ", offline: " + computer.isOffline()
+ ", offlineCause: " + (computer.getOfflineCause() != null ? computer.getOfflineCause().getClass() : "null"));
}
// call parent
super.afterDisconnect(computer, listener);
}
@Nonnull
@Override
public SubTask getParent() {
return dockerTask;
}
public synchronized int abortRunning(int number) throws IllegalAccessException {
int aborted = 0;
Computer[] computers = getJenkinsInstance().getComputers();
for (Computer computer : computers) {
if (isNull(computer)) {
continue;
}
List<Executor> executors = computer.getExecutors();
executors.addAll(computer.getOneOffExecutors());
for (Executor executor : executors) {
if (isNull(executor) || !executor.isBusy() || nonNull(executor.getCauseOfDeath()) ||
!getInterruptCauses(executor).isEmpty() || getInterruptStatus(executor) == Result.ABORTED) {
continue;
}
Queue.Executable executable = executor.getCurrentExecutable();
final SubTask parent = executable.getParent();
if (!(executable instanceof Run)) {
continue;
}
final Run executableRun = (Run) executable;
if (!(parent instanceof Job)) {
continue;
}
final Job parentJob = (Job) parent;
if (!parentJob.getFullName().equals(job.getFullName())) {
// name doesn't match
continue;
}
if (executableRun.getResult() == Result.ABORTED) {
// was already aborted
continue;
}
if (executableRun instanceof MatrixRun) {
// the whole MatrixBuild will be aborted
continue;
}
// if (executable instanceof MatrixBuild) {
// final MatrixBuild executable1 = (MatrixBuild) executable;
// executable1.doStop()
// }
final GitHubPRCause causeAction = (GitHubPRCause) executableRun.getCause(GitHubPRCause.class);
if (nonNull(causeAction) && causeAction.getNumber() == number) {
LOGGER.info("Aborting '{}', by interrupting '{}'", executableRun, executor);
executor.interrupt(Result.ABORTED, new NewPRInterruptCause());
aborted++;
}
}
}
return aborted;
}
@Override
public Label getAssignedLabel(@Nonnull SubTask task) {
return new LabelAtom(uuid);
}