Package org.jpmml.sparkml

Class DatasetUtil

  • public class DatasetUtil
extends Object

    • Method Summary

      All Methods Static Methods Concrete Methods 
      Modifier and Type Method Description
      static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumn​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, String name, org.apache.spark.sql.types.DataType sparkDataType)  
      static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumns​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, org.apache.spark.sql.types.StructType schema)  
      static org.apache.spark.sql.catalyst.plans.logical.LogicalPlan createAnalyzedLogicalPlan​(org.apache.spark.sql.SparkSession sparkSession, org.apache.spark.sql.types.StructType schema, String statement)  
      static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> loadCsv​(org.apache.spark.sql.SparkSession sparkSession, File file)  
      static org.apache.spark.sql.types.StructType loadSchema​(File file)  
      static void storeCsv​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, File file)  
      static void storeSchema​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, File file)  
      static void storeSchema​(org.apache.spark.sql.types.StructType schema, File file)  
      static org.dmg.pmml.DataType translateAtomicType​(org.apache.spark.sql.types.AtomicType atomicType)  
      static org.dmg.pmml.DataType translateDataType​(org.apache.spark.sql.types.DataType sparkDataType)  
      static org.dmg.pmml.DataType translateFractionalType​(org.apache.spark.sql.types.FractionalType fractionalType)  
      static org.dmg.pmml.DataType translateIntegralType​(org.apache.spark.sql.types.IntegralType integralType)  

    • Method Detail

      • loadSchema

        public static org.apache.spark.sql.types.StructType loadSchema​(File file)
                                                        throws IOException
        Throws:
        IOException

      • storeSchema

        public static void storeSchema​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                               File file)
                        throws IOException
        Throws:
        IOException

      • storeSchema

        public static void storeSchema​(org.apache.spark.sql.types.StructType schema,
                               File file)
                        throws IOException
        Throws:
        IOException

      • loadCsv

        public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> loadCsv​(org.apache.spark.sql.SparkSession sparkSession,
                                                                             File file)
                                                                      throws IOException
        Throws:
        IOException

      • storeCsv

        public static void storeCsv​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                            File file)
                     throws IOException
        Throws:
        IOException

      • castColumn

        public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumn​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                                                                                String name,
                                                                                org.apache.spark.sql.types.DataType sparkDataType)

      • castColumns

        public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> castColumns​(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                                                                                 org.apache.spark.sql.types.StructType schema)

      • createAnalyzedLogicalPlan

        public static org.apache.spark.sql.catalyst.plans.logical.LogicalPlan createAnalyzedLogicalPlan​(org.apache.spark.sql.SparkSession sparkSession,
                                                                                                org.apache.spark.sql.types.StructType schema,
                                                                                                String statement)

      • translateDataType

        public static org.dmg.pmml.DataType translateDataType​(org.apache.spark.sql.types.DataType sparkDataType)

      • translateAtomicType

        public static org.dmg.pmml.DataType translateAtomicType​(org.apache.spark.sql.types.AtomicType atomicType)

      • translateIntegralType

        public static org.dmg.pmml.DataType translateIntegralType​(org.apache.spark.sql.types.IntegralType integralType)

      • translateFractionalType

        public static org.dmg.pmml.DataType translateFractionalType​(org.apache.spark.sql.types.FractionalType fractionalType)