Java源码示例：org.apache.hadoop.yarn.client.api.NMClient
示例1
public SolrMaster(CommandLine cli) throws Exception {
this.cli = cli;
Configuration hadoopConf = new Configuration();
if (cli.hasOption("conf")) {
hadoopConf.addResource(new Path(cli.getOptionValue("conf")));
hadoopConf.reloadConfiguration();
}
conf = new YarnConfiguration(hadoopConf);
nmClient = NMClient.createNMClient();
nmClient.init(conf);
nmClient.start();
numContainersToWaitFor = Integer.parseInt(cli.getOptionValue("nodes"));
memory = Integer.parseInt(cli.getOptionValue("memory", "512"));
port = Integer.parseInt(cli.getOptionValue("port"));
nextPort = port;
SecureRandom random = new SecureRandom();
this.randomStopKey = new BigInteger(130, random).toString(32);
this.inetAddresses = getMyInetAddresses();
}
示例2
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
// create the client to communicate with the node managers
NMClient nodeManagerClient = NMClient.createNMClient();
nodeManagerClient.init(yarnConfiguration);
nodeManagerClient.start();
nodeManagerClient.cleanupRunningContainersOnStop(true);
return nodeManagerClient;
}
示例3
TestingYarnResourceManager(
RpcService rpcService,
String resourceManagerEndpointId,
ResourceID resourceId,
Configuration flinkConfig,
Map<String, String> env,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
SlotManager slotManager,
MetricRegistry metricRegistry,
JobLeaderIdService jobLeaderIdService,
ClusterInformation clusterInformation,
FatalErrorHandler fatalErrorHandler,
@Nullable String webInterfaceUrl,
AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
NMClient mockNMClient,
JobManagerMetricGroup jobManagerMetricGroup) {
super(
rpcService,
resourceManagerEndpointId,
resourceId,
flinkConfig,
env,
highAvailabilityServices,
heartbeatServices,
slotManager,
metricRegistry,
jobLeaderIdService,
clusterInformation,
fatalErrorHandler,
webInterfaceUrl,
jobManagerMetricGroup);
this.mockNMClient = mockNMClient;
this.mockResourceManagerClient = mockResourceManagerClient;
}
示例4
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
// create the client to communicate with the node managers
NMClient nodeManagerClient = NMClient.createNMClient();
nodeManagerClient.init(yarnConfiguration);
nodeManagerClient.start();
nodeManagerClient.cleanupRunningContainersOnStop(true);
return nodeManagerClient;
}
示例5
TestingYarnResourceManager(
RpcService rpcService,
String resourceManagerEndpointId,
ResourceID resourceId,
Configuration flinkConfig,
Map<String, String> env,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
SlotManager slotManager,
MetricRegistry metricRegistry,
JobLeaderIdService jobLeaderIdService,
ClusterInformation clusterInformation,
FatalErrorHandler fatalErrorHandler,
@Nullable String webInterfaceUrl,
AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
NMClient mockNMClient,
JobManagerMetricGroup jobManagerMetricGroup) {
super(
rpcService,
resourceManagerEndpointId,
resourceId,
flinkConfig,
env,
highAvailabilityServices,
heartbeatServices,
slotManager,
metricRegistry,
jobLeaderIdService,
clusterInformation,
fatalErrorHandler,
webInterfaceUrl,
jobManagerMetricGroup);
this.mockNMClient = mockNMClient;
this.mockResourceManagerClient = mockResourceManagerClient;
}
示例6
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
CallbackHandler callbackHandler) {
super(name, client, callbackHandler);
this.client = client;
this.callbackHandler = callbackHandler;
}
示例7
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
CallbackHandler callbackHandler) {
super(name);
this.setClient(client);
this.setCallbackHandler(callbackHandler);
}
示例8
private NMClient mockNMClient(int mode)
throws YarnException, IOException {
NMClient client = mock(NMClient.class);
switch (mode) {
case 0:
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.<String, ByteBuffer>emptyMap());
when(client.getContainerStatus(any(ContainerId.class),
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doNothing().when(client).stopContainer(any(ContainerId.class),
any(NodeId.class));
break;
case 1:
doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
.startContainer(any(Container.class),
any(ContainerLaunchContext.class));
doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
.getContainerStatus(any(ContainerId.class), any(NodeId.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
.stopContainer(any(ContainerId.class), any(NodeId.class));
break;
case 2:
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.<String, ByteBuffer>emptyMap());
when(client.getContainerStatus(any(ContainerId.class),
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
.stopContainer(any(ContainerId.class), any(NodeId.class));
}
return client;
}
示例9
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
CallbackHandler callbackHandler) {
super(name, client, callbackHandler);
this.client = client;
this.callbackHandler = callbackHandler;
}
示例10
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
CallbackHandler callbackHandler) {
super(name);
this.setClient(client);
this.setCallbackHandler(callbackHandler);
}
示例11
private NMClient mockNMClient(int mode)
throws YarnException, IOException {
NMClient client = mock(NMClient.class);
switch (mode) {
case 0:
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.<String, ByteBuffer>emptyMap());
when(client.getContainerStatus(any(ContainerId.class),
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doNothing().when(client).stopContainer(any(ContainerId.class),
any(NodeId.class));
break;
case 1:
doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
.startContainer(any(Container.class),
any(ContainerLaunchContext.class));
doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
.getContainerStatus(any(ContainerId.class), any(NodeId.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
.stopContainer(any(ContainerId.class), any(NodeId.class));
break;
case 2:
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.<String, ByteBuffer>emptyMap());
when(client.getContainerStatus(any(ContainerId.class),
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
.stopContainer(any(ContainerId.class), any(NodeId.class));
}
return client;
}
示例12
public boolean run() throws IOException, YarnException {
// Initialize clients to RM and NMs.
LOG.info("ApplicationMaster::run");
AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
resourceManager.init(conf);
resourceManager.start();
nodeManager = NMClient.createNMClient();
nodeManager.init(conf);
nodeManager.start();
// Register with RM
resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
appMasterTrackingUrl);
Log.info("total container count: "+Integer.toString(totalContainerCount));
// Ask RM to give us a bunch of containers
//for (int i = 0; i < totalContainerCount; i++) {
ContainerRequest containerReq = setupContainerReqForRM();
resourceManager.addContainerRequest(containerReq);
//}
requestedContainerCount.addAndGet(totalContainerCount);
while (!done) {
try {
Thread.sleep(200);
} catch (InterruptedException ex) {
}
}// while
// Un-register with ResourceManager
resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
return true;
}
示例13
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
logger.info("Start init(VMConfig vmConfig)");
this.vmConfig = vmConfig;
try {
this.yarnConfig = vmConfig.getHadoopProperties();
conf = new YarnConfiguration() ;
vmConfig.overrideHadoopConfiguration(conf);
amrmClient = AMRMClient.createAMRMClient();
amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
amrmClientAsync.init(conf);
amrmClientAsync.start();
nmClient = NMClient.createNMClient();
nmClient.init(conf);
nmClient.start();
// Register with RM
String appHostName = InetAddress.getLocalHost().getHostAddress() ;
RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
System.out.println("amrmClientAsync.registerApplicationMaster");
} catch(Throwable t) {
logger.error("Error: " , t);
t.printStackTrace();
}
logger.info("Finish init(VMConfig vmConfig)");
}
示例14
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
return mockNMClient;
}
示例15
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
return mockNMClient;
}
示例16
public NMClient getClient() {
return client;
}
示例17
public void setClient(NMClient client) {
this.client = client;
}
示例18
@Before
public void setup() throws YarnException, IOException {
// start minicluster
conf = new YarnConfiguration();
yarnCluster =
new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
assertNotNull(yarnCluster);
assertEquals(STATE.STARTED, yarnCluster.getServiceState());
// start rm client
yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
assertNotNull(yarnClient);
assertEquals(STATE.STARTED, yarnClient.getServiceState());
// get node info
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
// submit new app
ApplicationSubmissionContext appContext =
yarnClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application name
appContext.setApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.newInstance(0);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
// unmanaged AM
appContext.setUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Records
.newRecord(SubmitApplicationRequest.class);
appRequest.setApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.submitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0) {
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() ==
YarnApplicationState.ACCEPTED) {
attemptId = appReport.getCurrentApplicationAttemptId();
appAttempt =
yarnCluster.getResourceManager().getRMContext().getRMApps()
.get(attemptId.getApplicationId()).getCurrentAppAttempt();
while (true) {
if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
break;
}
}
break;
}
sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0) {
fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.setLoginUser(UserGroupInformation
.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
.<ContainerRequest> createAMRMClient();
//setting an instance NMTokenCase
rmClient.setNMTokenCache(nmTokenCache);
rmClient.init(conf);
rmClient.start();
assertNotNull(rmClient);
assertEquals(STATE.STARTED, rmClient.getServiceState());
// start am nm client
nmClient = (NMClientImpl) NMClient.createNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.setNMTokenCache(rmClient.getNMTokenCache());
nmClient.init(conf);
nmClient.start();
assertNotNull(nmClient);
assertEquals(STATE.STARTED, nmClient.getServiceState());
}
示例19
public Hadoop21YarnNMClient(Configuration configuration) {
this.nmClient = NMClient.createNMClient();
nmClient.init(configuration);
}
示例20
private ContainerTerminator(Container container, NMClient nmClient) {
this.container = container;
this.nmClient = nmClient;
}
示例21
public NMClient getClient() {
return client;
}
示例22
public void setClient(NMClient client) {
this.client = client;
}
示例23
@Before
public void setup() throws YarnException, IOException {
// start minicluster
conf = new YarnConfiguration();
yarnCluster =
new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
assertNotNull(yarnCluster);
assertEquals(STATE.STARTED, yarnCluster.getServiceState());
// start rm client
yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
assertNotNull(yarnClient);
assertEquals(STATE.STARTED, yarnClient.getServiceState());
// get node info
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
// submit new app
ApplicationSubmissionContext appContext =
yarnClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application name
appContext.setApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.newInstance(0);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
// unmanaged AM
appContext.setUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Records
.newRecord(SubmitApplicationRequest.class);
appRequest.setApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.submitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0) {
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() ==
YarnApplicationState.ACCEPTED) {
attemptId = appReport.getCurrentApplicationAttemptId();
appAttempt =
yarnCluster.getResourceManager().getRMContext().getRMApps()
.get(attemptId.getApplicationId()).getCurrentAppAttempt();
while (true) {
if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
break;
}
}
break;
}
sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0) {
fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.setLoginUser(UserGroupInformation
.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
.<ContainerRequest> createAMRMClient();
//setting an instance NMTokenCase
rmClient.setNMTokenCache(nmTokenCache);
rmClient.init(conf);
rmClient.start();
assertNotNull(rmClient);
assertEquals(STATE.STARTED, rmClient.getServiceState());
// start am nm client
nmClient = (NMClientImpl) NMClient.createNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.setNMTokenCache(rmClient.getNMTokenCache());
nmClient.init(conf);
nmClient.start();
assertNotNull(nmClient);
assertEquals(STATE.STARTED, nmClient.getServiceState());
}
示例24
/**
* Sets NMClient.
*
* @param nmClient NMClient.
*/
public void setNmClient(NMClient nmClient) {
this.nmClient = nmClient;
}
示例25
public NMClient getNMClient() { return this.nmClient ; }
