Java源码示例：org.apache.flink.runtime.rest.messages.job.UserAccumulator
示例1
@Override
protected JobVertexAccumulatorsInfo handleRequest(
HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request,
AccessExecutionJobVertex jobVertex) throws RestHandlerException {
StringifiedAccumulatorResult[] accs = jobVertex.getAggregatedUserAccumulatorsStringified();
ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(
new UserAccumulator(
acc.getName(),
acc.getType(),
acc.getValue()));
}
return new JobVertexAccumulatorsInfo(jobVertex.getJobVertexId().toString(), userAccumulatorList);
}
示例2
@Override
protected JobVertexAccumulatorsInfo getTestResponseInstance() throws Exception {
List<UserAccumulator> userAccumulatorList = new ArrayList<>(3);
userAccumulatorList.add(new UserAccumulator(
"test name1",
"test type1",
"test value1"));
userAccumulatorList.add(new UserAccumulator(
"test name2",
"test type2",
"test value2"));
userAccumulatorList.add(new UserAccumulator(
"test name3",
"test type3",
"test value3"));
return new JobVertexAccumulatorsInfo("testId", userAccumulatorList);
}
示例3
@Override
protected JobVertexAccumulatorsInfo handleRequest(
HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request,
AccessExecutionJobVertex jobVertex) throws RestHandlerException {
StringifiedAccumulatorResult[] accs = jobVertex.getAggregatedUserAccumulatorsStringified();
ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(
new UserAccumulator(
acc.getName(),
acc.getType(),
acc.getValue()));
}
return new JobVertexAccumulatorsInfo(jobVertex.getJobVertexId().toString(), userAccumulatorList);
}
示例4
@Override
protected JobVertexAccumulatorsInfo getTestResponseInstance() throws Exception {
List<UserAccumulator> userAccumulatorList = new ArrayList<>(3);
userAccumulatorList.add(new UserAccumulator(
"test name1",
"test type1",
"test value1"));
userAccumulatorList.add(new UserAccumulator(
"test name2",
"test type2",
"test value2"));
userAccumulatorList.add(new UserAccumulator(
"test name3",
"test type3",
"test value3"));
return new JobVertexAccumulatorsInfo("testId", userAccumulatorList);
}
示例5
@Override
protected JobVertexAccumulatorsInfo handleRequest(
HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request,
AccessExecutionJobVertex jobVertex) throws RestHandlerException {
StringifiedAccumulatorResult[] accs = jobVertex.getAggregatedUserAccumulatorsStringified();
ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(
new UserAccumulator(
acc.getName(),
acc.getType(),
acc.getValue()));
}
return new JobVertexAccumulatorsInfo(jobVertex.getJobVertexId().toString(), userAccumulatorList);
}
示例6
@Override
protected JobVertexAccumulatorsInfo getTestResponseInstance() throws Exception {
List<UserAccumulator> userAccumulatorList = new ArrayList<>(3);
userAccumulatorList.add(new UserAccumulator(
"test name1",
"test type1",
"test value1"));
userAccumulatorList.add(new UserAccumulator(
"test name2",
"test type2",
"test value2"));
userAccumulatorList.add(new UserAccumulator(
"test name3",
"test type3",
"test value3"));
return new JobVertexAccumulatorsInfo("testId", userAccumulatorList);
}
示例7
@JsonCreator
public JobVertexAccumulatorsInfo(
@JsonProperty(FIELD_NAME_ID) String id,
@JsonProperty(FIELD_NAME_USER_ACCUMULATORS) Collection<UserAccumulator> userAccumulatorList) {
this.id = Preconditions.checkNotNull(id);
this.userAccumulatorList = Preconditions.checkNotNull(userAccumulatorList);
}
示例8
@Override
protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
JobVertexID jobVertexId = jobVertex.getJobVertexId();
int parallelism = jobVertex.getParallelism();
final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>();
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
String locationString = location == null ? "(unassigned)" : location.getHostname();
StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
subtaskAccumulatorsInfos.add(
new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo(
vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(),
vertex.getCurrentExecutionAttempt().getAttemptNumber(),
locationString,
userAccumulators
));
}
return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos);
}
示例9
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
return new SubtaskExecutionAttemptAccumulatorsInfo(
execution.getParallelSubtaskIndex(),
execution.getAttemptNumber(),
execution.getAttemptId().toString(),
userAccumulatorList);
}
示例10
@JsonCreator
public JobVertexAccumulatorsInfo(
@JsonProperty(FIELD_NAME_ID) String id,
@JsonProperty(FIELD_NAME_USER_ACCUMULATORS) Collection<UserAccumulator> userAccumulatorList) {
this.id = Preconditions.checkNotNull(id);
this.userAccumulatorList = Preconditions.checkNotNull(userAccumulatorList);
}
示例11
@Override
protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
JobVertexID jobVertexId = jobVertex.getJobVertexId();
int parallelism = jobVertex.getParallelism();
final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>();
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
String locationString = location == null ? "(unassigned)" : location.getHostname();
StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
subtaskAccumulatorsInfos.add(
new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo(
vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(),
vertex.getCurrentExecutionAttempt().getAttemptNumber(),
locationString,
userAccumulators
));
}
return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos);
}
示例12
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
return new SubtaskExecutionAttemptAccumulatorsInfo(
execution.getParallelSubtaskIndex(),
execution.getAttemptNumber(),
execution.getAttemptId().toString(),
userAccumulatorList);
}
示例13
@JsonCreator
public JobVertexAccumulatorsInfo(
@JsonProperty(FIELD_NAME_ID) String id,
@JsonProperty(FIELD_NAME_USER_ACCUMULATORS) Collection<UserAccumulator> userAccumulatorList) {
this.id = Preconditions.checkNotNull(id);
this.userAccumulatorList = Preconditions.checkNotNull(userAccumulatorList);
}
示例14
@Override
protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
JobVertexID jobVertexId = jobVertex.getJobVertexId();
int parallelism = jobVertex.getParallelism();
final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>();
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
String locationString = location == null ? "(unassigned)" : location.getHostname();
StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
subtaskAccumulatorsInfos.add(
new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo(
vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(),
vertex.getCurrentExecutionAttempt().getAttemptNumber(),
locationString,
userAccumulators
));
}
return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos);
}
示例15
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
return new SubtaskExecutionAttemptAccumulatorsInfo(
execution.getParallelSubtaskIndex(),
execution.getAttemptNumber(),
execution.getAttemptId().toString(),
userAccumulatorList);
}
示例16
@Test
public void testHandleRequest() throws Exception {
// Instance the handler.
final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
final SubtaskExecutionAttemptAccumulatorsHandler handler = new SubtaskExecutionAttemptAccumulatorsHandler(
() -> null,
Time.milliseconds(100L),
Collections.emptyMap(),
SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(),
new ExecutionGraphCache(
restHandlerConfiguration.getTimeout(),
Time.milliseconds(restHandlerConfiguration.getRefreshInterval())),
TestingUtils.defaultExecutor());
// Instance a empty request.
final HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request = new HandlerRequest<>(
EmptyRequestBody.getInstance(),
new SubtaskAttemptMessageParameters()
);
final Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>(3);
userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10)));
userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L)));
userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
// Instance the expected result.
final StringifiedAccumulatorResult[] accumulatorResults =
StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators);
final int attemptNum = 1;
final int subtaskIndex = 2;
// Instance the tested execution.
final ArchivedExecution execution = new ArchivedExecution(
accumulatorResults,
null,
new ExecutionAttemptID(),
attemptNum,
ExecutionState.FINISHED,
null,
null,
null,
subtaskIndex,
new long[ExecutionState.values().length]);
// Invoke tested method.
final SubtaskExecutionAttemptAccumulatorsInfo accumulatorsInfo = handler.handleRequest(request, execution);
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(userAccumulators.size());
for (StringifiedAccumulatorResult accumulatorResult : accumulatorResults) {
userAccumulatorList.add(
new UserAccumulator(
accumulatorResult.getName(),
accumulatorResult.getType(),
accumulatorResult.getValue()));
}
final SubtaskExecutionAttemptAccumulatorsInfo expected = new SubtaskExecutionAttemptAccumulatorsInfo(
subtaskIndex,
attemptNum,
execution.getAttemptId().toString(),
userAccumulatorList);
// Verify.
assertEquals(expected, accumulatorsInfo);
}
示例17
@Test
public void testHandleRequest() throws Exception {
// Instance the handler.
final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
final SubtaskExecutionAttemptAccumulatorsHandler handler = new SubtaskExecutionAttemptAccumulatorsHandler(
() -> null,
Time.milliseconds(100L),
Collections.emptyMap(),
SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(),
new ExecutionGraphCache(
restHandlerConfiguration.getTimeout(),
Time.milliseconds(restHandlerConfiguration.getRefreshInterval())),
TestingUtils.defaultExecutor());
// Instance a empty request.
final HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request = new HandlerRequest<>(
EmptyRequestBody.getInstance(),
new SubtaskAttemptMessageParameters()
);
final Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>(3);
userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10)));
userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L)));
userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
// Instance the expected result.
final StringifiedAccumulatorResult[] accumulatorResults =
StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators);
final int attemptNum = 1;
final int subtaskIndex = 2;
// Instance the tested execution.
final ArchivedExecution execution = new ArchivedExecution(
accumulatorResults,
null,
new ExecutionAttemptID(),
attemptNum,
ExecutionState.FINISHED,
null,
null,
null,
subtaskIndex,
new long[ExecutionState.values().length]);
// Invoke tested method.
final SubtaskExecutionAttemptAccumulatorsInfo accumulatorsInfo = handler.handleRequest(request, execution);
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(userAccumulators.size());
for (StringifiedAccumulatorResult accumulatorResult : accumulatorResults) {
userAccumulatorList.add(
new UserAccumulator(
accumulatorResult.getName(),
accumulatorResult.getType(),
accumulatorResult.getValue()));
}
final SubtaskExecutionAttemptAccumulatorsInfo expected = new SubtaskExecutionAttemptAccumulatorsInfo(
subtaskIndex,
attemptNum,
execution.getAttemptId().toString(),
userAccumulatorList);
// Verify.
assertEquals(expected, accumulatorsInfo);
}
示例18
@Test
public void testHandleRequest() throws Exception {
// Instance the handler.
final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
final SubtaskExecutionAttemptAccumulatorsHandler handler = new SubtaskExecutionAttemptAccumulatorsHandler(
() -> null,
Time.milliseconds(100L),
Collections.emptyMap(),
SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(),
new DefaultExecutionGraphCache(
restHandlerConfiguration.getTimeout(),
Time.milliseconds(restHandlerConfiguration.getRefreshInterval())),
TestingUtils.defaultExecutor());
// Instance a empty request.
final HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request = new HandlerRequest<>(
EmptyRequestBody.getInstance(),
new SubtaskAttemptMessageParameters()
);
final Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>(3);
userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10)));
userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L)));
userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
// Instance the expected result.
final StringifiedAccumulatorResult[] accumulatorResults =
StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators);
final int attemptNum = 1;
final int subtaskIndex = 2;
// Instance the tested execution.
final ArchivedExecution execution = new ArchivedExecution(
accumulatorResults,
null,
new ExecutionAttemptID(),
attemptNum,
ExecutionState.FINISHED,
null,
null,
null,
subtaskIndex,
new long[ExecutionState.values().length]);
// Invoke tested method.
final SubtaskExecutionAttemptAccumulatorsInfo accumulatorsInfo = handler.handleRequest(request, execution);
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(userAccumulators.size());
for (StringifiedAccumulatorResult accumulatorResult : accumulatorResults) {
userAccumulatorList.add(
new UserAccumulator(
accumulatorResult.getName(),
accumulatorResult.getType(),
accumulatorResult.getValue()));
}
final SubtaskExecutionAttemptAccumulatorsInfo expected = new SubtaskExecutionAttemptAccumulatorsInfo(
subtaskIndex,
attemptNum,
execution.getAttemptId().toString(),
userAccumulatorList);
// Verify.
assertEquals(expected, accumulatorsInfo);
}
