StageMetrics

Stage Metrics: collects stage-level metrics with Stage granularity and provides aggregation and reporting functions for the end-user

Example usage for stage metrics: val stageMetrics = ch.cern.sparkmeasure.StageMetrics(spark) stageMetrics.runAndMeasure(spark.sql("select count(*) from range(1000) cross join range(1000) cross join range(1000)").show)

The tool is based on using Spark Listeners as data source and collecting metrics in a ListBuffer of a case class that encapsulates Spark task metrics. The List Buffer is then transformed into a DataFrame for ease of reporting and analysis.

Stage metrics are stored in memory and use to produce a report that aggregates resource consumption they can also be consumed "raw" (transformed into a DataFrame and/or saved to a file)

Linear Supertypes

Serializable, Serializable, Product, Equals, AnyRef, Any

Instance Constructors

new StageMetrics(sparkSession: SparkSession)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def aggregateStageMetrics(nameTempView: String = "PerfStageMetrics"): DataFrame
def aggregateStageMetrics(): LinkedHashMap[String, Long]
final def asInstanceOf[T0]: T0

Definition Classes
Any
def begin(): Long
var beginSnapshot: Long
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createStageMetricsDF(nameTempView: String = "PerfStageMetrics"): DataFrame
def end(): Long
var endSnapshot: Long
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val listenerStage: StageInfoRecorderListener
lazy val logger: Logger
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def printReport(): Unit
def registerListener(spark: SparkSession, listener: StageInfoRecorderListener): Unit
def removeListener(): Unit
def report(): String
def reportUsingDataFrame(): String
def runAndMeasure[T](f: ⇒ T): T
def saveData(df: DataFrame, fileName: String, fileFormat: String = "json", saveMode: String = "default"): Unit
def sendReportPrometheus(serverIPnPort: String, metricsJob: String, labelName: String = sparkSession.sparkContext.appName, labelValue: String = ...): Unit

Send the metrics to Prometheus.
Send the metrics to Prometheus. serverIPnPort: String with prometheus pushgateway address, format is hostIP:Port, metricsJob: job name, labelName: metrics label name, default is sparkSession.sparkContext.appName, labelValue: metrics label value, default is sparkSession.sparkContext.applicationId
val sparkSession: SparkSession
def stagesDuration(): LinkedHashMap[Int, Long]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package sparkmeasure

case class StageMetrics(sparkSession: SparkSession) extends Product with Serializable

Instance Constructors

new StageMetrics(sparkSession: SparkSession)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def aggregateStageMetrics(nameTempView: String = "PerfStageMetrics"): DataFrame

def aggregateStageMetrics(): LinkedHashMap[String, Long]

final def asInstanceOf[T0]: T0

def begin(): Long

var beginSnapshot: Long

def clone(): AnyRef

def createStageMetricsDF(nameTempView: String = "PerfStageMetrics"): DataFrame

def end(): Long

var endSnapshot: Long

final def eq(arg0: AnyRef): Boolean

def finalize(): Unit

final def getClass(): Class[_]

final def isInstanceOf[T0]: Boolean

val listenerStage: StageInfoRecorderListener

lazy val logger: Logger

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def printReport(): Unit

def registerListener(spark: SparkSession, listener: StageInfoRecorderListener): Unit

def removeListener(): Unit

def report(): String

def reportUsingDataFrame(): String

def runAndMeasure[T](f: ⇒ T): T

def saveData(df: DataFrame, fileName: String, fileFormat: String = "json", saveMode: String = "default"): Unit

def sendReportPrometheus(serverIPnPort: String, metricsJob: String, labelName: String = sparkSession.sparkContext.appName, labelValue: String = ...): Unit

val sparkSession: SparkSession

def stagesDuration(): LinkedHashMap[Int, Long]

final def synchronized[T0](arg0: ⇒ T0): T0

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped