c

ch.cern.sparkmeasure

StageInfoRecorderListener

class StageInfoRecorderListener extends SparkListener

StageInfoRecorderListener: this listener gathers metrics with Stage execution granularity It is based on the Spark Listener interface Stage metrics are stored in memory and use to produce a report that aggregates resource consumption they can also be consumed "raw" (transformed into a DataFrame and/or saved to a file) See StageMetrics

Linear Supertypes
SparkListener, SparkListenerInterface, AnyRef, Any
Known Subclasses
FlightRecorderStageMetrics
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. StageInfoRecorderListener
  2. SparkListener
  3. SparkListenerInterface
  4. AnyRef
  5. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new StageInfoRecorderListener(stageInfoVerbose: Boolean, executorMetricNames: Array[String])

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @IntrinsicCandidate()
  6. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  7. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  8. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @IntrinsicCandidate()
  9. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @IntrinsicCandidate()
  10. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  11. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  12. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @IntrinsicCandidate()
  13. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @IntrinsicCandidate()
  14. def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  15. def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  16. def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  17. def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  18. def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  19. def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  20. def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  21. def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  22. def onExecutorBlacklistedForStage(executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  23. def onExecutorExcluded(executorExcluded: SparkListenerExecutorExcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  24. def onExecutorExcludedForStage(executorExcludedForStage: SparkListenerExecutorExcludedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  25. def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit
    Definition Classes
    StageInfoRecorderListener → SparkListener → SparkListenerInterface
  26. def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  27. def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  28. def onExecutorUnexcluded(executorUnexcluded: SparkListenerExecutorUnexcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  29. def onJobEnd(jobEnd: SparkListenerJobEnd): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  30. def onJobStart(jobStart: SparkListenerJobStart): Unit
    Definition Classes
    StageInfoRecorderListener → SparkListener → SparkListenerInterface
  31. def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  32. def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  33. def onNodeExcluded(nodeExcluded: SparkListenerNodeExcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  34. def onNodeExcludedForStage(nodeExcludedForStage: SparkListenerNodeExcludedForStage): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  35. def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  36. def onNodeUnexcluded(nodeUnexcluded: SparkListenerNodeUnexcluded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  37. def onOtherEvent(event: SparkListenerEvent): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  38. def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  39. def onSpeculativeTaskSubmitted(speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  40. def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

    This methods fires at the end of the stage and collects metrics flattened into the stageMetricsData ListBuffer Note all reported times are in ms, cpu time and shuffle write time are originally in nanoseconds, thus in the code are divided by 1 million to normalize them to milliseconds

    This methods fires at the end of the stage and collects metrics flattened into the stageMetricsData ListBuffer Note all reported times are in ms, cpu time and shuffle write time are originally in nanoseconds, thus in the code are divided by 1 million to normalize them to milliseconds

    Definition Classes
    StageInfoRecorderListener → SparkListener → SparkListenerInterface
  41. def onStageExecutorMetrics(executorMetrics: SparkListenerStageExecutorMetrics): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  42. def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  43. def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  44. def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  45. def onTaskStart(taskStart: SparkListenerTaskStart): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  46. def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  47. def onUnschedulableTaskSetAdded(unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  48. def onUnschedulableTaskSetRemoved(unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit
    Definition Classes
    SparkListener → SparkListenerInterface
  49. val stageIdtoExecutorMetrics: HashMap[(Int, String), ListBuffer[(String, Long)]]
  50. val stageIdtoJobGroup: HashMap[Int, String]
  51. val stageIdtoJobId: HashMap[Int, Int]
  52. val stageMetricsData: ListBuffer[StageVals]
  53. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  54. def toString(): String
    Definition Classes
    AnyRef → Any
  55. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  56. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  57. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated
    Deprecated

Inherited from SparkListener

Inherited from SparkListenerInterface

Inherited from AnyRef

Inherited from Any

Ungrouped