class StageInfoRecorderListener extends SparkListener
StageInfoRecorderListener: this listener gathers metrics with Stage execution granularity It is based on the Spark Listener interface Stage metrics are stored in memory and use to produce a report that aggregates resource consumption they can also be consumed "raw" (transformed into a DataFrame and/or saved to a file) See StageMetrics
Linear Supertypes
Known Subclasses
Ordering
- Alphabetic
- By Inheritance
Inherited
- StageInfoRecorderListener
- SparkListener
- SparkListenerInterface
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- Protected
Instance Constructors
- new StageInfoRecorderListener(stageInfoVerbose: Boolean, executorMetricNames: Array[String])
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @native() @HotSpotIntrinsicCandidate()
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def equals(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef → Any
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- def hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorBlacklistedForStage(executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorExcluded(executorExcluded: SparkListenerExecutorExcluded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorExcludedForStage(executorExcludedForStage: SparkListenerExecutorExcludedForStage): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit
- Definition Classes
- StageInfoRecorderListener → SparkListener → SparkListenerInterface
- def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onExecutorUnexcluded(executorUnexcluded: SparkListenerExecutorUnexcluded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onJobEnd(jobEnd: SparkListenerJobEnd): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onJobStart(jobStart: SparkListenerJobStart): Unit
- Definition Classes
- StageInfoRecorderListener → SparkListener → SparkListenerInterface
- def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onNodeExcluded(nodeExcluded: SparkListenerNodeExcluded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onNodeExcludedForStage(nodeExcludedForStage: SparkListenerNodeExcludedForStage): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onNodeUnexcluded(nodeUnexcluded: SparkListenerNodeUnexcluded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onOtherEvent(event: SparkListenerEvent): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onSpeculativeTaskSubmitted(speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit
This methods fires at the end of the stage and collects metrics flattened into the stageMetricsData ListBuffer Note all reported times are in ms, cpu time and shuffle write time are originally in nanoseconds, thus in the code are divided by 1 million to normalize them to milliseconds
This methods fires at the end of the stage and collects metrics flattened into the stageMetricsData ListBuffer Note all reported times are in ms, cpu time and shuffle write time are originally in nanoseconds, thus in the code are divided by 1 million to normalize them to milliseconds
- Definition Classes
- StageInfoRecorderListener → SparkListener → SparkListenerInterface
- def onStageExecutorMetrics(executorMetrics: SparkListenerStageExecutorMetrics): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onTaskStart(taskStart: SparkListenerTaskStart): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onUnschedulableTaskSetAdded(unschedulableTaskSetAdded: SparkListenerUnschedulableTaskSetAdded): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- def onUnschedulableTaskSetRemoved(unschedulableTaskSetRemoved: SparkListenerUnschedulableTaskSetRemoved): Unit
- Definition Classes
- SparkListener → SparkListenerInterface
- val stageIdtoExecutorMetrics: HashMap[(Int, String), ListBuffer[(String, Long)]]
- val stageIdtoJobGroup: HashMap[Int, String]
- val stageIdtoJobId: HashMap[Int, Int]
- val stageMetricsData: ListBuffer[StageVals]
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toString(): String
- Definition Classes
- AnyRef → Any
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
Deprecated Value Members
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable]) @Deprecated
- Deprecated