case class StageVals(jobId: Int, jobGroup: String, stageId: Int, name: String, submissionTime: Long, completionTime: Long, stageDuration: Long, numTasks: Int, executorRunTime: Long, executorCpuTime: Long, executorDeserializeTime: Long, executorDeserializeCpuTime: Long, resultSerializationTime: Long, jvmGCTime: Long, resultSize: Long, numUpdatedBlockStatuses: Int, diskBytesSpilled: Long, memoryBytesSpilled: Long, peakExecutionMemory: Long, recordsRead: Long, bytesRead: Long, recordsWritten: Long, bytesWritten: Long, shuffleFetchWaitTime: Long, shuffleTotalBytesRead: Long, shuffleTotalBlocksFetched: Long, shuffleLocalBlocksFetched: Long, shuffleRemoteBlocksFetched: Long, shuffleWriteTime: Long, shuffleBytesWritten: Long, shuffleRecordsWritten: Long) extends Product with Serializable

Stage Metrics: collects and aggregates metrics at the end of each stage Task Metrics: collects data at task granularity

Example usage for stage metrics: val stageMetrics = ch.cern.sparkmeasure.StageMetrics(spark) stageMetrics.runAndMeasure(spark.sql("select count(*) from range(1000) cross join range(1000) cross join range(1000)").show)

The tool is based on using Spark Listeners as data source and collecting metrics in a ListBuffer of a case class that encapsulates Spark task metrics. The List Buffer is then transformed into a DataFrame for ease of reporting and analysis.

Linear Supertypes
Serializable, Serializable, Product, Equals, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. StageVals
  2. Serializable
  3. Serializable
  4. Product
  5. Equals
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new StageVals(jobId: Int, jobGroup: String, stageId: Int, name: String, submissionTime: Long, completionTime: Long, stageDuration: Long, numTasks: Int, executorRunTime: Long, executorCpuTime: Long, executorDeserializeTime: Long, executorDeserializeCpuTime: Long, resultSerializationTime: Long, jvmGCTime: Long, resultSize: Long, numUpdatedBlockStatuses: Int, diskBytesSpilled: Long, memoryBytesSpilled: Long, peakExecutionMemory: Long, recordsRead: Long, bytesRead: Long, recordsWritten: Long, bytesWritten: Long, shuffleFetchWaitTime: Long, shuffleTotalBytesRead: Long, shuffleTotalBlocksFetched: Long, shuffleLocalBlocksFetched: Long, shuffleRemoteBlocksFetched: Long, shuffleWriteTime: Long, shuffleBytesWritten: Long, shuffleRecordsWritten: Long)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. val bytesRead: Long
  6. val bytesWritten: Long
  7. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  8. val completionTime: Long
  9. val diskBytesSpilled: Long
  10. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  11. val executorCpuTime: Long
  12. val executorDeserializeCpuTime: Long
  13. val executorDeserializeTime: Long
  14. val executorRunTime: Long
  15. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  16. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  17. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  18. val jobGroup: String
  19. val jobId: Int
  20. val jvmGCTime: Long
  21. val memoryBytesSpilled: Long
  22. val name: String
  23. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  24. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  25. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  26. val numTasks: Int
  27. val numUpdatedBlockStatuses: Int
  28. val peakExecutionMemory: Long
  29. val recordsRead: Long
  30. val recordsWritten: Long
  31. val resultSerializationTime: Long
  32. val resultSize: Long
  33. val shuffleBytesWritten: Long
  34. val shuffleFetchWaitTime: Long
  35. val shuffleLocalBlocksFetched: Long
  36. val shuffleRecordsWritten: Long
  37. val shuffleRemoteBlocksFetched: Long
  38. val shuffleTotalBlocksFetched: Long
  39. val shuffleTotalBytesRead: Long
  40. val shuffleWriteTime: Long
  41. val stageDuration: Long
  42. val stageId: Int
  43. val submissionTime: Long
  44. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  45. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  46. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  47. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped