org.jpmml.sparkml

Class DatasetUtil

java.lang.Object

org.jpmml.sparkml.DatasetUtil

public class DatasetUtil
extends Object

Method Summary

Modifier and Type	Method and Description
static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	castColumn(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, String name, org.apache.spark.sql.types.DataType sparkDataType)
static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	castColumns(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, org.apache.spark.sql.types.StructType schema)
static org.apache.spark.sql.catalyst.plans.logical.LogicalPlan	createAnalyzedLogicalPlan(org.apache.spark.sql.SparkSession sparkSession, org.apache.spark.sql.types.StructType schema, String statement)
static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	loadCsv(org.apache.spark.sql.SparkSession sparkSession, File file)
static org.apache.spark.sql.types.StructType	loadSchema(File file)
static void	storeCsv(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, File file)
static void	storeSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, File file)
static void	storeSchema(org.apache.spark.sql.types.StructType schema, File file)
static org.dmg.pmml.DataType	translateAtomicType(org.apache.spark.sql.types.AtomicType atomicType)
static org.dmg.pmml.DataType	translateDataType(org.apache.spark.sql.types.DataType sparkDataType)
static org.dmg.pmml.DataType	translateFractionalType(org.apache.spark.sql.types.FractionalType fractionalType)
static org.dmg.pmml.DataType	translateIntegralType(org.apache.spark.sql.types.IntegralType integralType)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

loadSchema

public static org.apache.spark.sql.types.StructType loadSchema(File file)
                                                        throws IOException

Throws:

IOException

storeSchema

public static void storeSchema(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                               File file)
                        throws IOException

Throws:

IOException

storeSchema

public static void storeSchema(org.apache.spark.sql.types.StructType schema,
                               File file)
                        throws IOException

Throws:

IOException

loadCsv

public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> loadCsv(org.apache.spark.sql.SparkSession sparkSession,
                                                                             File file)
                                                                      throws IOException

Throws:

IOException

storeCsv

public static void storeCsv(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                            File file)
                     throws IOException

Throws:

IOException

castColumn

public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumn(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                                                                                String name,
                                                                                org.apache.spark.sql.types.DataType sparkDataType)

castColumns

public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumns(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                                                                                 org.apache.spark.sql.types.StructType schema)

createAnalyzedLogicalPlan

public static org.apache.spark.sql.catalyst.plans.logical.LogicalPlan createAnalyzedLogicalPlan(org.apache.spark.sql.SparkSession sparkSession,
                                                                                                org.apache.spark.sql.types.StructType schema,
                                                                                                String statement)

translateDataType

public static org.dmg.pmml.DataType translateDataType(org.apache.spark.sql.types.DataType sparkDataType)

translateAtomicType

public static org.dmg.pmml.DataType translateAtomicType(org.apache.spark.sql.types.AtomicType atomicType)

translateIntegralType

public static org.dmg.pmml.DataType translateIntegralType(org.apache.spark.sql.types.IntegralType integralType)

translateFractionalType

public static org.dmg.pmml.DataType translateFractionalType(org.apache.spark.sql.types.FractionalType fractionalType)