class KafkaSink extends SparkListener

KafkaSink: write Spark metrics and application info in near real-time to Kafka stream use this mode to monitor Spark execution workload use for Grafana dashboard and analytics of job execution

How to use: attach the KafkaSink to a Spark Context using the extra listener infrastructure. Example: --conf spark.extraListeners=ch.cern.sparkmeasure.KafkaSink

Configuration for KafkaSink is handled with Spark conf parameters:

spark.sparkmeasure.kafkaBroker = Kafka broker endpoint URL example: --conf spark.sparkmeasure.kafkaBroker=kafka.your-site.com:9092 spark.sparkmeasure.kafkaTopic = Kafka topic example: --conf spark.sparkmeasure.kafkaTopic=sparkmeasure-stageinfo

This code depends on "kafka clients", you may need to add the dependency: --packages org.apache.kafka:kafka-clients:3.2.1

Output: each message contains the name, it is acknowledged as metrics name as well. Note: the amount of data generated is relatively small in most applications: O(number_of_stages)

Linear Supertypes
SparkListener, SparkListenerInterface, AnyRef, Any
Known Subclasses
KafkaSinkExtended
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. KafkaSink
  2. SparkListener
  3. SparkListenerInterface
  4. AnyRef
  5. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new KafkaSink(conf: SparkConf)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. var appId: String
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. val broker: String
  7. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @IntrinsicCandidate()
  8. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  9. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  10. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @IntrinsicCandidate()
  11. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @IntrinsicCandidate()
  12. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  13. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  14. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @IntrinsicCandidate()
  15. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @IntrinsicCandidate()
  16. def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  17. def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  18. def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  19. def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  20. def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  21. def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  22. def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  23. def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  24. def onExecutorBlacklistedForStage(executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  25. def onExecutorExcluded(executorExcluded: SparkListenerExecutorExcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  26. def onExecutorExcludedForStage(executorExcludedForStage: SparkListenerExecutorExcludedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  27. def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  28. def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  29. def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  30. def onExecutorUnexcluded(executorUnexcluded: SparkListenerExecutorUnexcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  31. def onJobEnd(jobEnd: SparkListenerJobEnd): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  32. def onJobStart(jobStart: SparkListenerJobStart): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  33. def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  34. def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  35. def onNodeExcluded(nodeExcluded: SparkListenerNodeExcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  36. def onNodeExcludedForStage(nodeExcludedForStage: SparkListenerNodeExcludedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  37. def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  38. def onNodeUnexcluded(nodeUnexcluded: SparkListenerNodeUnexcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  39. def onOtherEvent(event: SparkListenerEvent): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  40. def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  41. def onSpeculativeTaskSubmitted(speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  42. def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  43. def onStageExecutorMetrics(executorMetrics: SparkListenerStageExecutorMetrics): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  44. def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit
    Definition Classes
    KafkaSink → SparkListener → SparkListenerInterface
  45. def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  46. def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  47. def onTaskStart(taskStart: SparkListenerTaskStart): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  48. def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  49. def onUnschedulableTaskSetAdded(unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  50. def onUnschedulableTaskSetRemoved(unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  51. def report[T](metrics: Map[String, T]): Unit
    Attributes
    protected
  52. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  53. def toString(): String
    Definition Classes
    AnyRef → Any
  54. val topic: String
  55. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  56. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  57. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated
    Deprecated

Inherited from SparkListener

Inherited from SparkListenerInterface

Inherited from AnyRef

Inherited from Any

Ungrouped