Package org.gorpipe.spark

Class GeneralSparkCluster

java.lang.Object

org.gorpipe.spark.platform.GorClusterBase

org.gorpipe.spark.GeneralSparkCluster

All Implemented Interfaces:

java.lang.AutoCloseable, java.lang.Runnable

public final class GeneralSparkCluster
extends GorClusterBase

Nested Class Summary

Nested classes/interfaces inherited from class org.gorpipe.spark.platform.GorClusterBase

GorClusterBase.Statistics

Constructor Summary

Constructors

Constructor	Description
GeneralSparkCluster(java.lang.String sparkRedisUri)

Method Summary

Modifier and Type	Method	Description
GorLogSubscription	addLogSubscription(GorLogReceiver receiver, java.lang.String... jobIds)
void	addScore(java.lang.String partition, int seconds)
void	close()
org.gorpipe.gor.model.GorParallelQueryHandler	createQueryHandler(boolean workOff, GorQuery query, java.lang.String securityContext)
java.lang.Object	createWorker()
java.lang.Object	createWorker(java.lang.String queue)
GorJob	findJob(java.lang.String jobId)
GorJob	findJobByFingerprint(java.lang.String fingerprint, java.util.function.Function<java.net.URI,GorClusterBase> clusterLookup)
protected void	forget(GorTaskBase task)
GorClusterConfig	getConfig()
GorClusterBase.Statistics	getCurrentStatistics()
java.time.Duration	getJobExpiration()
java.lang.String	getJobMessage(java.lang.String requestId)
java.time.Duration	getJobRetention()
java.util.Map<JobStatus,java.lang.Integer>	getJobStatusesByRequestId(java.lang.String requestId)
java.lang.String	getSecurityContext(java.lang.String key)
java.lang.String	getValue(java.lang.String jobId, JobField field)
boolean	jobExists(java.lang.String jobId)
GorLocks	locks()
void	logDebug(java.lang.String message)
void	logError(java.lang.String message, java.lang.Throwable th)
void	logInfo(java.lang.String message, java.lang.Throwable ex)
void	logProgress(java.lang.String jobId, java.lang.String progress)
void	logWarn(java.lang.String message, java.lang.Throwable ex)
redis.clients.jedis.JedisPool	pool()
GorJob	prioritySubmit(java.lang.String queue, java.lang.String task, java.lang.Object... args)
void	registerJobByFingerprint(java.lang.String fingerprint, GorJob job)
void	run()
void	setValue(java.lang.String jobId, JobField field, java.lang.String value)
java.lang.String	storeSecurityContext(java.lang.String securityContext)
GorJob	submit(java.lang.String queue, java.lang.String task, java.lang.Object... args)
java.util.List<GorJob>	submit(java.lang.String queue, BatchSubmission submission)
void	subscribeGorMonitor(java.lang.String requestId, org.gorpipe.gor.monitor.GorMonitor gorMonitor)
void	yieldFor(java.util.Collection<GorJob> jobs, java.lang.String queue, JobMonitor monitor)

Methods inherited from class org.gorpipe.spark.platform.GorClusterBase

cancel, getLocalHostName, onWorkerException, waitFor, workerFatalException

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

GeneralSparkCluster

public GeneralSparkCluster(java.lang.String sparkRedisUri)

Method Detail

getJobExpiration

public java.time.Duration getJobExpiration()

Specified by:

getJobExpiration in class GorClusterBase

getConfig

public GorClusterConfig getConfig()

Specified by:

getConfig in class GorClusterBase

logInfo

public void logInfo(java.lang.String message,
                    java.lang.Throwable ex)

Specified by:

logInfo in class GorClusterBase

logDebug

public void logDebug(java.lang.String message)

Specified by:

logDebug in class GorClusterBase

logWarn

public void logWarn(java.lang.String message,
                    java.lang.Throwable ex)

Specified by:

logWarn in class GorClusterBase

logError

public void logError(java.lang.String message,
                     java.lang.Throwable th)

Specified by:

logError in class GorClusterBase

logProgress

public void logProgress(java.lang.String jobId,
                        java.lang.String progress)

Specified by:

logProgress in class GorClusterBase

getValue

public java.lang.String getValue(java.lang.String jobId,
                                 JobField field)

Specified by:

getValue in class GorClusterBase

setValue

public void setValue(java.lang.String jobId,
                     JobField field,
                     java.lang.String value)

Specified by:

setValue in class GorClusterBase

yieldFor

public void yieldFor(java.util.Collection<GorJob> jobs,
                     java.lang.String queue,
                     JobMonitor monitor)

Specified by:

yieldFor in class GorClusterBase

locks

public GorLocks locks()

Specified by:

locks in class GorClusterBase

pool

public redis.clients.jedis.JedisPool pool()

Specified by:

pool in class GorClusterBase

addLogSubscription

public GorLogSubscription addLogSubscription(GorLogReceiver receiver,
                                             java.lang.String... jobIds)

Specified by:

addLogSubscription in class GorClusterBase

findJob

public GorJob findJob(java.lang.String jobId)

Specified by:

findJob in class GorClusterBase

getJobStatusesByRequestId

public java.util.Map<JobStatus,java.lang.Integer> getJobStatusesByRequestId(java.lang.String requestId)

Specified by:

getJobStatusesByRequestId in class GorClusterBase

getJobMessage

public java.lang.String getJobMessage(java.lang.String requestId)

Specified by:

getJobMessage in class GorClusterBase

subscribeGorMonitor

public void subscribeGorMonitor(java.lang.String requestId,
                                org.gorpipe.gor.monitor.GorMonitor gorMonitor)

Specified by:

subscribeGorMonitor in class GorClusterBase

jobExists

public boolean jobExists(java.lang.String jobId)

Specified by:

jobExists in class GorClusterBase

findJobByFingerprint

public GorJob findJobByFingerprint(java.lang.String fingerprint,
                                   java.util.function.Function<java.net.URI,GorClusterBase> clusterLookup)

Specified by:

findJobByFingerprint in class GorClusterBase

registerJobByFingerprint

public void registerJobByFingerprint(java.lang.String fingerprint,
                                     GorJob job)

Specified by:

registerJobByFingerprint in class GorClusterBase

forget

protected void forget(GorTaskBase task)

Specified by:

forget in class GorClusterBase

getCurrentStatistics

public GorClusterBase.Statistics getCurrentStatistics()

Specified by:

getCurrentStatistics in class GorClusterBase

submit

public GorJob submit(java.lang.String queue,
                     java.lang.String task,
                     java.lang.Object... args)

Specified by:

submit in class GorClusterBase

submit

public java.util.List<GorJob> submit(java.lang.String queue,
                                     BatchSubmission submission)

Specified by:

submit in class GorClusterBase

prioritySubmit

public GorJob prioritySubmit(java.lang.String queue,
                             java.lang.String task,
                             java.lang.Object... args)

Specified by:

prioritySubmit in class GorClusterBase

addScore

public void addScore(java.lang.String partition,
                     int seconds)

Specified by:

addScore in class GorClusterBase

storeSecurityContext

public java.lang.String storeSecurityContext(java.lang.String securityContext)

Specified by:

storeSecurityContext in class GorClusterBase

getSecurityContext

public java.lang.String getSecurityContext(java.lang.String key)

Specified by:

getSecurityContext in class GorClusterBase

createQueryHandler

public org.gorpipe.gor.model.GorParallelQueryHandler createQueryHandler(boolean workOff,
                                                                        GorQuery query,
                                                                        java.lang.String securityContext)

Specified by:

createQueryHandler in class GorClusterBase

createWorker

public java.lang.Object createWorker()

Specified by:

createWorker in class GorClusterBase

createWorker

public java.lang.Object createWorker(java.lang.String queue)

Specified by:

createWorker in class GorClusterBase

close

public void close()

run

public void run()

getJobRetention

public java.time.Duration getJobRetention()

Specified by:

getJobRetention in class GorClusterBase