ml.shifu.guagua.yarn

Class GuaguaAppMaster

java.lang.Object

ml.shifu.guagua.yarn.GuaguaAppMaster

public class GuaguaAppMaster
extends Object

GuaguaAppMaster is application master to launch master and worker tasks.

This app master is used to check and launch all tasks, not to run the master task of distributed training. Master task is run on another container.

TODO: Web monitor is not supported in current app master.

Fail-over is added like mapreduce. If one container is failed, it will be launched until 4 times by default.

In each container, use a number start from 1 to mark as the id of the container for fail-over.

Constructor Summary

Constructors

Constructor and Description
GuaguaAppMaster(org.apache.hadoop.yarn.api.records.ContainerId cId, org.apache.hadoop.yarn.api.records.ApplicationAttemptId aId, org.apache.hadoop.conf.Configuration conf) Construct the GuaguappMaster, populate fields using env vars and set up by YARN framework in this execution container.

Method Summary

Modifier and Type	Method and Description
AtomicInteger	getAllocatedCount()
org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest>	getAmRMClient()
org.apache.hadoop.yarn.api.records.ApplicationAttemptId	getAppAttemptId()
org.apache.hadoop.yarn.api.records.ApplicationId	getAppId()
AtomicInteger	getCompletedCount()
String	getContainerArgs()
org.apache.hadoop.yarn.api.records.ContainerId	getContainerId()
ml.shifu.guagua.yarn.GuaguaAppMaster.NMCallbackHandler	getContainerListener()
int	getContainersToLaunch()
ExecutorService	getExecutor()
AtomicInteger	getFailedCount()
int	getHeapPerContainer()
List<org.apache.hadoop.mapreduce.InputSplit>	getNewSplits(org.apache.hadoop.conf.Configuration conf)
org.apache.hadoop.yarn.client.api.async.NMClientAsync	getNmClientAsync()
AtomicInteger	getSuccessfulCount()
org.apache.hadoop.conf.Configuration	getYarnConf()
boolean	isDone()
static void	main(String[] args) Application entry point
boolean	run() Coordinates all requests for guagua's worker/master task containers, and manages application liveness heartbeat, completion status, teardown, etc.
void	setAllocatedCount(AtomicInteger allocatedCount)
void	setAmRMClient(org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> amRMClient)
void	setAppAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
void	setAppId(org.apache.hadoop.yarn.api.records.ApplicationId appId)
void	setCompletedCount(AtomicInteger completedCount)
void	setContainerArgs(String containerArgs)
void	setContainerId(org.apache.hadoop.yarn.api.records.ContainerId containerId)
void	setContainerListener(ml.shifu.guagua.yarn.GuaguaAppMaster.NMCallbackHandler containerListener)
void	setContainersToLaunch(int containersToLaunch)
void	setDone(boolean done)
void	setExecutor(ExecutorService executor)
void	setFailedCount(AtomicInteger failedCount)
void	setHeapPerContainer(int heapPerContainer)
void	setNmClientAsync(org.apache.hadoop.yarn.client.api.async.NMClientAsync nmClientAsync)
void	setSuccessfulCount(AtomicInteger successfulCount)
void	setYarnConf(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConf)
protected void	shutdown()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

GuaguaAppMaster

public GuaguaAppMaster(org.apache.hadoop.yarn.api.records.ContainerId cId,
                       org.apache.hadoop.yarn.api.records.ApplicationAttemptId aId,
                       org.apache.hadoop.conf.Configuration conf)

Construct the GuaguappMaster, populate fields using env vars and set up by YARN framework in this execution container.

Parameters:

cId - the ContainerId

aId - the ApplicationAttemptId

Method Detail

run

public boolean run()
            throws org.apache.hadoop.yarn.exceptions.YarnException,
                   IOException

Coordinates all requests for guagua's worker/master task containers, and manages application liveness heartbeat, completion status, teardown, etc.

Returns:

success or failure

Throws:

org.apache.hadoop.yarn.exceptions.YarnException

IOException

shutdown

protected void shutdown()

getNewSplits

public List<org.apache.hadoop.mapreduce.InputSplit> getNewSplits(org.apache.hadoop.conf.Configuration conf)
                                                          throws IOException

Throws:

IOException

getContainerId

public org.apache.hadoop.yarn.api.records.ContainerId getContainerId()

setContainerId

public void setContainerId(org.apache.hadoop.yarn.api.records.ContainerId containerId)

getAppAttemptId

public org.apache.hadoop.yarn.api.records.ApplicationAttemptId getAppAttemptId()

setAppAttemptId

public void setAppAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)

isDone

public boolean isDone()

setDone

public void setDone(boolean done)

getYarnConf

public org.apache.hadoop.conf.Configuration getYarnConf()

setYarnConf

public void setYarnConf(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConf)

getCompletedCount

public AtomicInteger getCompletedCount()

setCompletedCount

public void setCompletedCount(AtomicInteger completedCount)

getFailedCount

public AtomicInteger getFailedCount()

setFailedCount

public void setFailedCount(AtomicInteger failedCount)

getAllocatedCount

public AtomicInteger getAllocatedCount()

setAllocatedCount

public void setAllocatedCount(AtomicInteger allocatedCount)

getSuccessfulCount

public AtomicInteger getSuccessfulCount()

setSuccessfulCount

public void setSuccessfulCount(AtomicInteger successfulCount)

getContainersToLaunch

public int getContainersToLaunch()

setContainersToLaunch

public void setContainersToLaunch(int containersToLaunch)

getExecutor

public ExecutorService getExecutor()

setExecutor

public void setExecutor(ExecutorService executor)

getHeapPerContainer

public int getHeapPerContainer()

setHeapPerContainer

public void setHeapPerContainer(int heapPerContainer)

getAmRMClient

public org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> getAmRMClient()

setAmRMClient

public void setAmRMClient(org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> amRMClient)

getNmClientAsync

public org.apache.hadoop.yarn.client.api.async.NMClientAsync getNmClientAsync()

setNmClientAsync

public void setNmClientAsync(org.apache.hadoop.yarn.client.api.async.NMClientAsync nmClientAsync)

getContainerListener

public ml.shifu.guagua.yarn.GuaguaAppMaster.NMCallbackHandler getContainerListener()

setContainerListener

public void setContainerListener(ml.shifu.guagua.yarn.GuaguaAppMaster.NMCallbackHandler containerListener)

getContainerArgs

public String getContainerArgs()

setContainerArgs

public void setContainerArgs(String containerArgs)

getAppId

public org.apache.hadoop.yarn.api.records.ApplicationId getAppId()

setAppId

public void setAppId(org.apache.hadoop.yarn.api.records.ApplicationId appId)

main

public static void main(String[] args)

Application entry point

Parameters:

args - command-line args (set by GuaguaYarnClient, if any)