Package gorsat.process

Class SparkRowUtilities

java.lang.Object

gorsat.process.SparkRowUtilities

public class SparkRowUtilities
extends Object

Constructor Summary

Constructors

Constructor	Description
SparkRowUtilities()

Method Summary

Modifier and Type	Method	Description
static List<String>	createMapList(Map<String,String> createMap, Map<String,String> defMap, String creates)
static String	createMapString(Map<String,String> createMap, Map<String,String> defMap, String creates)
static String	generateTempViewName(String fileName, boolean usegorpipe, String filter, String chr, int pos, int end)
static String	generateTempViewName(String fileName, boolean usegorpipe, String filter, String chr, int pos, int end, List<Instant> inst)
static Predicate<String>	getFileEndingPredicate()
static Predicate<String>	getFileEndingTimestampPredicate()
static org.apache.spark.sql.types.StructType	gor2Schema(String header, org.gorpipe.gor.model.Row types)
static GorDataType	gorCmdSchema(String[] gorcmds, org.gorpipe.spark.GorSparkSession gorSparkSession)
static GorDataType	gorCmdSchema(String gorcmd, org.gorpipe.spark.GorSparkSession gorSparkSession, boolean nor)
static org.apache.spark.sql.types.StructType	gorDataTypeToStructType(GorDataType gorDataType)
static GorDataType	inferDataTypes(InputStream is, String fileName, boolean isGorz, boolean nor)
static GorDataType	inferDataTypes(org.gorpipe.gor.model.FileReader fileReader, String fileName, boolean isGorz, boolean nor)
static org.apache.spark.sql.types.StructType	inferSchema(InputStream fileStream, String fileName, boolean nor, boolean isGorz)
static org.apache.spark.sql.Dataset<? extends org.apache.spark.sql.Row>	registerFile(String[] fns, String name, String profile, org.gorpipe.spark.GorSparkSession gorSparkSession, String standalone, String cacheDir, boolean usestreaming, String filter, String filterFile, String filterColumn, String splitFile, boolean nor, String chr, int pos, int end, String jobid, String cacheFile, boolean cpp, boolean tag, org.apache.spark.sql.types.StructType schema, Map<String,String> readOptions)
static RowDataType	translatePath(String fn, String standalone, org.gorpipe.gor.model.DriverBackedFileReader fileReader)
static GorDataType	typeFromParquetLine(org.gorpipe.gor.model.ParquetLine pl, boolean withStart, String[] header)
static GorDataType	typeFromStream(Stream<String> linestream, boolean withStart, String[] headerArray, boolean nor)
static GorDataType	typeFromStream(Stream<String> linestream, boolean withStart, String[] headerArray, boolean nor, boolean base128)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

SparkRowUtilities

public SparkRowUtilities()

Method Details

getFileEndingPredicate

public static Predicate<String> getFileEndingPredicate()

getFileEndingTimestampPredicate

public static Predicate<String> getFileEndingTimestampPredicate()

createMapString

public static String createMapString(Map<String,String> createMap,
 Map<String,String> defMap,
 String creates)

createMapList

public static List<String> createMapList(Map<String,String> createMap,
 Map<String,String> defMap,
 String creates)

generateTempViewName

public static String generateTempViewName(String fileName,
 boolean usegorpipe,
 String filter,
 String chr,
 int pos,
 int end)

generateTempViewName

public static String generateTempViewName(String fileName,
 boolean usegorpipe,
 String filter,
 String chr,
 int pos,
 int end,
 List<Instant> inst)

gor2Schema

public static org.apache.spark.sql.types.StructType gor2Schema(String header,
 org.gorpipe.gor.model.Row types)

gorDataTypeToStructType

public static org.apache.spark.sql.types.StructType gorDataTypeToStructType(GorDataType gorDataType)

inferSchema

public static org.apache.spark.sql.types.StructType inferSchema(InputStream fileStream,
 String fileName,
 boolean nor,
 boolean isGorz)
                                                         throws IOException,
DataFormatException

Throws:

IOException

DataFormatException

translatePath

public static RowDataType translatePath(String fn,
 String standalone,
 org.gorpipe.gor.model.DriverBackedFileReader fileReader)
                                 throws IOException

Throws:

IOException

gorCmdSchema

public static GorDataType gorCmdSchema(String gorcmd,
 org.gorpipe.spark.GorSparkSession gorSparkSession,
 boolean nor)

gorCmdSchema

public static GorDataType gorCmdSchema(String[] gorcmds,
 org.gorpipe.spark.GorSparkSession gorSparkSession)

registerFile

public static org.apache.spark.sql.Dataset<? extends org.apache.spark.sql.Row> registerFile(String[] fns,
 String name,
 String profile,
 org.gorpipe.spark.GorSparkSession gorSparkSession,
 String standalone,
 String cacheDir,
 boolean usestreaming,
 String filter,
 String filterFile,
 String filterColumn,
 String splitFile,
 boolean nor,
 String chr,
 int pos,
 int end,
 String jobid,
 String cacheFile,
 boolean cpp,
 boolean tag,
 org.apache.spark.sql.types.StructType schema,
 Map<String,String> readOptions)
                                                                                     throws IOException,
DataFormatException

Throws:

IOException

DataFormatException

inferDataTypes

public static GorDataType inferDataTypes(org.gorpipe.gor.model.FileReader fileReader,
 String fileName,
 boolean isGorz,
 boolean nor)
                                  throws IOException,
DataFormatException

Throws:

IOException

DataFormatException

inferDataTypes

public static GorDataType inferDataTypes(InputStream is,
 String fileName,
 boolean isGorz,
 boolean nor)
                                  throws IOException,
DataFormatException

Throws:

IOException

DataFormatException

typeFromParquetLine

public static GorDataType typeFromParquetLine(org.gorpipe.gor.model.ParquetLine pl,
 boolean withStart,
 String[] header)

typeFromStream

public static GorDataType typeFromStream(Stream<String> linestream,
 boolean withStart,
 String[] headerArray,
 boolean nor)

typeFromStream

public static GorDataType typeFromStream(Stream<String> linestream,
 boolean withStart,
 String[] headerArray,
 boolean nor,
 boolean base128)