ai.databand

Class DbndWrapper

java.lang.Object

ai.databand.DbndWrapper

public class DbndWrapper
extends java.lang.Object

AspectJ wrapper for @Pipeline and @Task annonations.

Constructor Summary

Constructors

Constructor and Description
DbndWrapper()

Method Summary

Modifier and Type	Method and Description
void	afterPipeline()
void	afterTask(java.lang.String methodName, java.lang.Object result)
void	beforePipeline(java.lang.String className, java.lang.String methodName, java.lang.Object[] args)
void	beforeTask(java.lang.String className, java.lang.String methodName, java.lang.Object[] args)
protected void	cleanup()
DbndConfig	config()
protected DbndRun	currentRun()
void	errorPipeline(java.lang.Throwable error)
void	errorTask(java.lang.String methodName, java.lang.Throwable error)
protected java.lang.reflect.Method	findMethodByName(java.lang.String methodName, java.lang.String classname)
static DbndWrapper	instance()
java.util.Optional<java.lang.Class<?>>	loadClass(java.lang.String className)
protected void	loadMethods(java.lang.String classname)
void	logDataframe(java.lang.String key, org.apache.spark.sql.Dataset<?> value, boolean withHistograms)
void	logDataframe(java.lang.String key, org.apache.spark.sql.Dataset<?> value, HistogramRequest histogramRequest)
void	logDatasetOperation(java.lang.String path, DatasetOperationType type, DatasetOperationStatus status, org.apache.spark.sql.Dataset<?> data, java.lang.Throwable error, LogDatasetRequest params)
void	logDatasetOperation(java.lang.String path, DatasetOperationType type, DatasetOperationStatus status, java.lang.String valuePreview, java.util.List<java.lang.Long> dataDimensions, java.lang.String dataSchema, java.lang.Boolean withPartition, java.util.List<ColumnStats> columnStats)
void	logHistogram(java.util.Map<java.lang.String,java.lang.Object> histogram)
void	logMetric(java.lang.String key, java.lang.Object value)
void	logMetrics(java.util.Map<java.lang.String,java.lang.Object> metrics)
void	logMetrics(java.util.Map<java.lang.String,java.lang.Object> metrics, java.lang.String source)
void	logSpark(org.apache.spark.scheduler.SparkListenerEvent event)
void	logTask(org.apache.log4j.spi.LoggingEvent event, java.lang.String eventStr)
protected void	printStack()
protected java.lang.String	removeArgsFromMethodName(java.lang.String methodName) Removes arguments part from string representation of method name.
void	setExternalTaskContext(java.lang.String runUid, java.lang.String taskRunUid, java.lang.String taskRunAttemptUid, java.lang.String taskName) Set tracking context from external source.
protected void	stop()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

DbndWrapper

public DbndWrapper()

Method Detail

instance

public static DbndWrapper instance()

loadClass

public java.util.Optional<java.lang.Class<?>> loadClass(java.lang.String className)

beforePipeline

public void beforePipeline(java.lang.String className,
                           java.lang.String methodName,
                           java.lang.Object[] args)

findMethodByName

protected java.lang.reflect.Method findMethodByName(java.lang.String methodName,
                                                    java.lang.String classname)

removeArgsFromMethodName

protected java.lang.String removeArgsFromMethodName(java.lang.String methodName)

Removes arguments part from string representation of method name. ai.databand.JavaSparkPipeline.execute(java.lang.String) → ai.databand.JavaSparkPipeline.execute( Opening parent should be present in result because latter it will be used in methods cache calculation

Parameters:

methodName -

Returns:

loadMethods

protected void loadMethods(java.lang.String classname)

afterPipeline

public void afterPipeline()

errorPipeline

public void errorPipeline(java.lang.Throwable error)

cleanup

protected void cleanup()

beforeTask

public void beforeTask(java.lang.String className,
                       java.lang.String methodName,
                       java.lang.Object[] args)

afterTask

public void afterTask(java.lang.String methodName,
                      java.lang.Object result)

errorTask

public void errorTask(java.lang.String methodName,
                      java.lang.Throwable error)

logTask

public void logTask(org.apache.log4j.spi.LoggingEvent event,
                    java.lang.String eventStr)

logMetric

public void logMetric(java.lang.String key,
                      java.lang.Object value)

logDatasetOperation

public void logDatasetOperation(java.lang.String path,
                                DatasetOperationType type,
                                DatasetOperationStatus status,
                                org.apache.spark.sql.Dataset<?> data,
                                java.lang.Throwable error,
                                LogDatasetRequest params)

logDatasetOperation

public void logDatasetOperation(java.lang.String path,
                                DatasetOperationType type,
                                DatasetOperationStatus status,
                                java.lang.String valuePreview,
                                java.util.List<java.lang.Long> dataDimensions,
                                java.lang.String dataSchema,
                                java.lang.Boolean withPartition,
                                java.util.List<ColumnStats> columnStats)

logMetrics

public void logMetrics(java.util.Map<java.lang.String,java.lang.Object> metrics)

logMetrics

public void logMetrics(java.util.Map<java.lang.String,java.lang.Object> metrics,
                       java.lang.String source)

logDataframe

public void logDataframe(java.lang.String key,
                         org.apache.spark.sql.Dataset<?> value,
                         HistogramRequest histogramRequest)

logHistogram

public void logHistogram(java.util.Map<java.lang.String,java.lang.Object> histogram)

logDataframe

public void logDataframe(java.lang.String key,
                         org.apache.spark.sql.Dataset<?> value,
                         boolean withHistograms)

logSpark

public void logSpark(org.apache.spark.scheduler.SparkListenerEvent event)

config

public DbndConfig config()

stop

protected void stop()

currentRun

protected DbndRun currentRun()

printStack

protected void printStack()

setExternalTaskContext

public void setExternalTaskContext(java.lang.String runUid,
                                   java.lang.String taskRunUid,
                                   java.lang.String taskRunAttemptUid,
                                   java.lang.String taskName)

Set tracking context from external source. This allows us to set context externally (for instance when calling pyspark script) and avoid runs duplication. TODO: since context can be controlled externally in this way, it may sense to start/stop JVM tasks from the Python

Parameters:

runUid -

taskRunUid -

taskRunAttemptUid -

taskName -