case class TaskVals(jobId: Int, jobGroup: String, stageId: Int, index: Long, launchTime: Long, finishTime: Long, duration: Long, schedulerDelay: Long, executorId: String, host: String, taskLocality: Int, speculative: Boolean, gettingResultTime: Long, successful: Boolean, executorRunTime: Long, executorCpuTime: Long, executorDeserializeTime: Long, executorDeserializeCpuTime: Long, resultSerializationTime: Long, jvmGCTime: Long, resultSize: Long, numUpdatedBlockStatuses: Int, diskBytesSpilled: Long, memoryBytesSpilled: Long, peakExecutionMemory: Long, recordsRead: Long, bytesRead: Long, recordsWritten: Long, bytesWritten: Long, shuffleFetchWaitTime: Long, shuffleTotalBytesRead: Long, shuffleTotalBlocksFetched: Long, shuffleLocalBlocksFetched: Long, shuffleRemoteBlocksFetched: Long, shuffleWriteTime: Long, shuffleBytesWritten: Long, shuffleRecordsWritten: Long) extends Product with Serializable

Stage Metrics: collects and aggregates metrics at the end of each stage Task Metrics: collects data at task granularity

Example usage for task metrics: val taskMetrics = ch.cern.sparkmeasure.TaskMetrics(spark) taskMetrics.runAndMeasure(spark.sql("select count(*) from range(1000) cross join range(1000) cross join range(1000)").show)

The tool is based on using Spark Listeners as data source and collecting metrics in a ListBuffer of a case class that encapsulates Spark task metrics. The List Buffer is then transformed into a DataFrame for ease of reporting and analysis.

Linear Supertypes
Serializable, Serializable, Product, Equals, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. TaskVals
  2. Serializable
  3. Serializable
  4. Product
  5. Equals
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new TaskVals(jobId: Int, jobGroup: String, stageId: Int, index: Long, launchTime: Long, finishTime: Long, duration: Long, schedulerDelay: Long, executorId: String, host: String, taskLocality: Int, speculative: Boolean, gettingResultTime: Long, successful: Boolean, executorRunTime: Long, executorCpuTime: Long, executorDeserializeTime: Long, executorDeserializeCpuTime: Long, resultSerializationTime: Long, jvmGCTime: Long, resultSize: Long, numUpdatedBlockStatuses: Int, diskBytesSpilled: Long, memoryBytesSpilled: Long, peakExecutionMemory: Long, recordsRead: Long, bytesRead: Long, recordsWritten: Long, bytesWritten: Long, shuffleFetchWaitTime: Long, shuffleTotalBytesRead: Long, shuffleTotalBlocksFetched: Long, shuffleLocalBlocksFetched: Long, shuffleRemoteBlocksFetched: Long, shuffleWriteTime: Long, shuffleBytesWritten: Long, shuffleRecordsWritten: Long)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. val bytesRead: Long
  6. val bytesWritten: Long
  7. def clone(): AnyRef
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @native() @throws( ... )
  8. val diskBytesSpilled: Long
  9. val duration: Long
  10. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  11. val executorCpuTime: Long
  12. val executorDeserializeCpuTime: Long
  13. val executorDeserializeTime: Long
  14. val executorId: String
  15. val executorRunTime: Long
  16. def finalize(): Unit
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  17. val finishTime: Long
  18. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  19. val gettingResultTime: Long
  20. val host: String
  21. val index: Long
  22. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  23. val jobGroup: String
  24. val jobId: Int
  25. val jvmGCTime: Long
  26. val launchTime: Long
  27. val memoryBytesSpilled: Long
  28. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  29. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  30. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  31. val numUpdatedBlockStatuses: Int
  32. val peakExecutionMemory: Long
  33. val recordsRead: Long
  34. val recordsWritten: Long
  35. val resultSerializationTime: Long
  36. val resultSize: Long
  37. val schedulerDelay: Long
  38. val shuffleBytesWritten: Long
  39. val shuffleFetchWaitTime: Long
  40. val shuffleLocalBlocksFetched: Long
  41. val shuffleRecordsWritten: Long
  42. val shuffleRemoteBlocksFetched: Long
  43. val shuffleTotalBlocksFetched: Long
  44. val shuffleTotalBytesRead: Long
  45. val shuffleWriteTime: Long
  46. val speculative: Boolean
  47. val stageId: Int
  48. val successful: Boolean
  49. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  50. val taskLocality: Int
  51. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  52. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  53. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @throws( ... )

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped