java.lang.Object

org.gorpipe.spark.platform.GorClusterBase

org.gorpipe.spark.GeneralSparkCluster

All Implemented Interfaces:: java.lang.AutoCloseable, java.lang.Runnable

public final class GeneralSparkCluster
extends GorClusterBase

Nested Class Summary

Nested classes/interfaces inherited from class org.gorpipe.spark.platform.GorClusterBase
GorClusterBase.Statistics
Constructor Summary

Constructors

Constructor Description

GeneralSparkCluster(java.lang.String sparkRedisUri)

Method Summary

Modifier and Type	Method	Description
`GorLogSubscription`	`addLogSubscription(GorLogReceiver receiver, java.lang.String... jobIds)`
`void`	`addScore(java.lang.String partition, int seconds)`
`void`	`close()`
`org.gorpipe.gor.model.GorParallelQueryHandler`	`createQueryHandler(boolean workOff, GorQuery query, java.lang.String securityContext)`
`java.lang.Object`	`createWorker()`
`java.lang.Object`	`createWorker(java.lang.String queue)`
`GorJob`	`findJob(java.lang.String jobId)`
`GorJob`	`findJobByFingerprint(java.lang.String fingerprint, java.util.function.Function<java.net.URI,GorClusterBase> clusterLookup)`
`protected void`	`forget(GorTaskBase task)`
`GorClusterConfig`	`getConfig()`
`GorClusterBase.Statistics`	`getCurrentStatistics()`
`java.time.Duration`	`getJobExpiration()`
`java.lang.String`	`getJobMessage(java.lang.String requestId)`
`java.time.Duration`	`getJobRetention()`
`java.util.Map<JobStatus,java.lang.Integer>`	`getJobStatusesByRequestId(java.lang.String requestId)`
`java.lang.String`	`getSecurityContext(java.lang.String key)`
`java.lang.String`	`getValue(java.lang.String jobId, JobField field)`
`boolean`	`jobExists(java.lang.String jobId)`
`GorLocks`	`locks()`
`void`	`logDebug(java.lang.String message)`
`void`	`logError(java.lang.String message, java.lang.Throwable th)`
`void`	`logInfo(java.lang.String message, java.lang.Throwable ex)`
`void`	`logProgress(java.lang.String jobId, java.lang.String progress)`
`void`	`logWarn(java.lang.String message, java.lang.Throwable ex)`
`redis.clients.jedis.JedisPool`	`pool()`
`GorJob`	`prioritySubmit(java.lang.String queue, java.lang.String task, java.lang.Object... args)`
`void`	`registerJobByFingerprint(java.lang.String fingerprint, GorJob job)`
`void`	`run()`
`void`	`setValue(java.lang.String jobId, JobField field, java.lang.String value)`
`java.lang.String`	`storeSecurityContext(java.lang.String securityContext)`
`GorJob`	`submit(java.lang.String queue, java.lang.String task, java.lang.Object... args)`
`java.util.List<GorJob>`	`submit(java.lang.String queue, BatchSubmission submission)`
`void`	`subscribeGorMonitor(java.lang.String requestId, org.gorpipe.gor.monitor.GorMonitor gorMonitor)`
`void`	`yieldFor(java.util.Collection<GorJob> jobs, java.lang.String queue, JobMonitor monitor)`

Methods inherited from class org.gorpipe.spark.platform.GorClusterBase

cancel, getLocalHostName, onWorkerException, waitFor, workerFatalException

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details
- GeneralSparkCluster
  
  public GeneralSparkCluster(java.lang.String sparkRedisUri)
Method Details
- getJobExpiration
  
  public java.time.Duration getJobExpiration()
  
  Specified by:
  
  getJobExpiration in class GorClusterBase
- getConfig
  
  public GorClusterConfig getConfig()
  
  Specified by:
  
  getConfig in class GorClusterBase
- logInfo
  
  public void logInfo(java.lang.String message, java.lang.Throwable ex)
  
  Specified by:
  
  logInfo in class GorClusterBase
- logDebug
  
  public void logDebug(java.lang.String message)
  
  Specified by:
  
  logDebug in class GorClusterBase
- logWarn
  
  public void logWarn(java.lang.String message, java.lang.Throwable ex)
  
  Specified by:
  
  logWarn in class GorClusterBase
- logError
  
  public void logError(java.lang.String message, java.lang.Throwable th)
  
  Specified by:
  
  logError in class GorClusterBase
- logProgress
  
  public void logProgress(java.lang.String jobId, java.lang.String progress)
  
  Specified by:
  
  logProgress in class GorClusterBase
- getValue
  
  public java.lang.String getValue(java.lang.String jobId, JobField field)
  
  Specified by:
  
  getValue in class GorClusterBase
- setValue
  
  public void setValue(java.lang.String jobId, JobField field, java.lang.String value)
  
  Specified by:
  
  setValue in class GorClusterBase
- yieldFor
  
  public void yieldFor(java.util.Collection<GorJob> jobs, java.lang.String queue, JobMonitor monitor)
  
  Specified by:
  
  yieldFor in class GorClusterBase
- locks
  
  public GorLocks locks()
  
  Specified by:
  
  locks in class GorClusterBase
- pool
  
  public redis.clients.jedis.JedisPool pool()
  
  Specified by:
  
  pool in class GorClusterBase
- addLogSubscription
  
  public GorLogSubscription addLogSubscription(GorLogReceiver receiver, java.lang.String... jobIds)
  
  Specified by:
  
  addLogSubscription in class GorClusterBase
- findJob
  
  public GorJob findJob(java.lang.String jobId)
  
  Specified by:
  
  findJob in class GorClusterBase
- getJobStatusesByRequestId
  
  public java.util.Map<JobStatus,java.lang.Integer> getJobStatusesByRequestId(java.lang.String requestId)
  
  Specified by:
  
  getJobStatusesByRequestId in class GorClusterBase
- getJobMessage
  
  public java.lang.String getJobMessage(java.lang.String requestId)
  
  Specified by:
  
  getJobMessage in class GorClusterBase
- subscribeGorMonitor
  
  public void subscribeGorMonitor(java.lang.String requestId, org.gorpipe.gor.monitor.GorMonitor gorMonitor)
  
  Specified by:
  
  subscribeGorMonitor in class GorClusterBase
- jobExists
  
  public boolean jobExists(java.lang.String jobId)
  
  Specified by:
  
  jobExists in class GorClusterBase
- findJobByFingerprint
  
  public GorJob findJobByFingerprint(java.lang.String fingerprint, java.util.function.Function<java.net.URI,GorClusterBase> clusterLookup)
  
  Specified by:
  
  findJobByFingerprint in class GorClusterBase
- registerJobByFingerprint
  
  public void registerJobByFingerprint(java.lang.String fingerprint, GorJob job)
  
  Specified by:
  
  registerJobByFingerprint in class GorClusterBase
- forget
  
  protected void forget(GorTaskBase task)
  
  Specified by:
  
  forget in class GorClusterBase
- getCurrentStatistics
  
  public GorClusterBase.Statistics getCurrentStatistics()
  
  Specified by:
  
  getCurrentStatistics in class GorClusterBase
- submit
  
  public GorJob submit(java.lang.String queue, java.lang.String task, java.lang.Object... args)
  
  Specified by:
  
  submit in class GorClusterBase
- submit
  
  public java.util.List<GorJob> submit(java.lang.String queue, BatchSubmission submission)
  
  Specified by:
  
  submit in class GorClusterBase
- prioritySubmit
  
  public GorJob prioritySubmit(java.lang.String queue, java.lang.String task, java.lang.Object... args)
  
  Specified by:
  
  prioritySubmit in class GorClusterBase
- addScore
  
  public void addScore(java.lang.String partition, int seconds)
  
  Specified by:
  
  addScore in class GorClusterBase
- storeSecurityContext
  
  public java.lang.String storeSecurityContext(java.lang.String securityContext)
  
  Specified by:
  
  storeSecurityContext in class GorClusterBase
- getSecurityContext
  
  public java.lang.String getSecurityContext(java.lang.String key)
  
  Specified by:
  
  getSecurityContext in class GorClusterBase
- createQueryHandler
  
  public org.gorpipe.gor.model.GorParallelQueryHandler createQueryHandler(boolean workOff, GorQuery query, java.lang.String securityContext)
  
  Specified by:
  
  createQueryHandler in class GorClusterBase
- createWorker
  
  public java.lang.Object createWorker()
  
  Specified by:
  
  createWorker in class GorClusterBase
- createWorker
  
  public java.lang.Object createWorker(java.lang.String queue)
  
  Specified by:
  
  createWorker in class GorClusterBase
- close
  
  public void close()
- run
  
  public void run()
- getJobRetention
  
  public java.time.Duration getJobRetention()
  
  Specified by:
  
  getJobRetention in class GorClusterBase

Class GeneralSparkCluster

Nested Class Summary

Nested classes/interfaces inherited from class org.gorpipe.spark.platform.GorClusterBase

Constructor Summary

Method Summary

Methods inherited from class org.gorpipe.spark.platform.GorClusterBase

Methods inherited from class java.lang.Object

Constructor Details

Method Details