org.bytedeco.arrow.global

Class arrow_dataset

java.lang.Object

org.bytedeco.arrow.presets.arrow_dataset

org.bytedeco.arrow.global.arrow_dataset

All Implemented Interfaces:

InfoMapper

public class arrow_dataset
extends arrow_dataset

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	arrow_dataset.ExistingDataBehavior \brief Controls what happens if files exist in an output directory during a dataset write
static class	arrow_dataset.SegmentEncoding \brief The encoding of partition segments.

Nested classes/interfaces inherited from class org.bytedeco.arrow.presets.arrow_dataset

arrow_dataset.CustomOpenWithCompression, arrow_dataset.EnumeratedRecordBatchGenerator, arrow_dataset.FileWriterVisitor, arrow_dataset.RecordBatchGenerator, arrow_dataset.RecordBatchVisitor, arrow_dataset.TaggedRecordBatchGenerator, arrow_dataset.TaggedRecordBatchVisitor

Constructor Summary

Constructors

Constructor and Description
arrow_dataset()

Method Summary

Modifier and Type	Method and Description
static Status	CheckProjectable(Schema from, Schema to) Logic for automatically determining the structure of multi-file dataset with possible partitioning according to available partitioning
static void	DieWithMessage(BytePointer msg)
static void	DieWithMessage(String msg) \brief If the generator is empty, return the given value, else forward the values from the generator.
static void	InitializeDatasetWriter(ExecFactoryRegistry registry) \}
static void	InitializeScanner(ExecFactoryRegistry registry) \}
static void	InvalidValueOrDie(Status st)
static String	kCsvTypeName()
static byte	kCsvTypeName(int i)
static int	kDefaultBackgroundMaxQ()
static int	kDefaultBackgroundQRestart()
static int	kDefaultBackpressureHigh()
static int	kDefaultBackpressureLow()
static int	kDefaultBatchReadahead()
static long	kDefaultBatchSize() \defgroup dataset-scanning Scanning API \{
static int	kDefaultFragmentReadahead()
static String	kDefaultHiveNullFallback()
static byte	kDefaultHiveNullFallback(int i) \brief The default fallback used for null values in a Hive-style partitioning.
static String	kIpcTypeName()
static byte	kIpcTypeName(int i) \addtogroup dataset-file-formats \{
static String	kParquetTypeName()
static byte	kParquetTypeName(int i) \addtogroup dataset-file-formats \{
static Pointer	shiftLeft(Pointer os, arrow_dataset.SegmentEncoding segment_encoding)
static Pointer	shiftLeft(Pointer os, byte segment_encoding)
static BytePointer	StripPrefixAndFilename(BytePointer path, BytePointer prefix)
static StringVector	StripPrefixAndFilename(FileInfoVector files, BytePointer prefix)
static StringVector	StripPrefixAndFilename(FileInfoVector files, String prefix) \brief Vector version of StripPrefixAndFilename.
static String	StripPrefixAndFilename(String path, String prefix) \brief Remove a prefix and the filename of a path.
static StringVector	StripPrefixAndFilename(StringVector paths, BytePointer prefix)
static StringVector	StripPrefixAndFilename(StringVector paths, String prefix) \brief Vector version of StripPrefixAndFilename.

Methods inherited from class org.bytedeco.arrow.presets.arrow_dataset

map

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

arrow_dataset

public arrow_dataset()

Method Detail

kDefaultBackgroundMaxQ

@Namespace(value="arrow")
 @MemberGetter
public static int kDefaultBackgroundMaxQ()

See Also:

MakeBackgroundGenerator

kDefaultBackgroundQRestart

@Namespace(value="arrow")
 @MemberGetter
public static int kDefaultBackgroundQRestart()

DieWithMessage

@Namespace(value="arrow::internal")
public static void DieWithMessage(@StdString
                                                                       String msg)

\brief If the generator is empty, return the given value, else forward the values from the generator. This generator is async-reentrant.

DieWithMessage

@Namespace(value="arrow::internal")
public static void DieWithMessage(@StdString
                                                                       BytePointer msg)

InvalidValueOrDie

@Namespace(value="arrow::internal")
public static void InvalidValueOrDie(@Const @ByRef
                                                                          Status st)

shiftLeft

@Namespace(value="arrow::dataset")
 @Cast(value="std::ostream*")
 @ByRef
 @Name(value="operator <<")
public static Pointer shiftLeft(@Cast(value="std::ostream*") @ByRef
                                                                                                                                         Pointer os,
                                                                                                                                         arrow_dataset.SegmentEncoding segment_encoding)

shiftLeft

@Namespace(value="arrow::dataset")
 @Cast(value="std::ostream*")
 @ByRef
 @Name(value="operator <<")
public static Pointer shiftLeft(@Cast(value="std::ostream*") @ByRef
                                                                                                                                         Pointer os,
                                                                                                                                         @Cast(value="arrow::dataset::SegmentEncoding")
                                                                                                                                         byte segment_encoding)

kDefaultHiveNullFallback

@Namespace(value="arrow::dataset")
 @MemberGetter
public static byte kDefaultHiveNullFallback(int i)

\brief The default fallback used for null values in a Hive-style partitioning.

kDefaultHiveNullFallback

@Namespace(value="arrow::dataset")
 @MemberGetter
public static String kDefaultHiveNullFallback()

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @StdString
public static String StripPrefixAndFilename(@StdString
                                                                                             String path,
                                                                                             @StdString
                                                                                             String prefix)

\brief Remove a prefix and the filename of a path. e.g., StripPrefixAndFilename("/data/year=2019/c.txt", "/data") -> "year=2019"

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @StdString
public static BytePointer StripPrefixAndFilename(@StdString
                                                                                                  BytePointer path,
                                                                                                  @StdString
                                                                                                  BytePointer prefix)

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @ByVal
public static StringVector StripPrefixAndFilename(@Const @ByRef
                                                                                               StringVector paths,
                                                                                               @StdString
                                                                                               String prefix)

\brief Vector version of StripPrefixAndFilename.

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @ByVal
public static StringVector StripPrefixAndFilename(@Const @ByRef
                                                                                               StringVector paths,
                                                                                               @StdString
                                                                                               BytePointer prefix)

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @ByVal
public static StringVector StripPrefixAndFilename(@Const @ByRef
                                                                                               FileInfoVector files,
                                                                                               @StdString
                                                                                               String prefix)

\brief Vector version of StripPrefixAndFilename.

StripPrefixAndFilename

@Namespace(value="arrow::dataset")
 @ByVal
public static StringVector StripPrefixAndFilename(@Const @ByRef
                                                                                               FileInfoVector files,
                                                                                               @StdString
                                                                                               BytePointer prefix)

CheckProjectable

@Namespace(value="arrow::dataset")
 @ByVal
public static Status CheckProjectable(@Const @ByRef
                                                                                   Schema from,
                                                                                   @Const @ByRef
                                                                                   Schema to)

Logic for automatically determining the structure of multi-file dataset with possible partitioning according to available partitioning

kDefaultBatchSize

@Namespace(value="arrow::dataset")
 @MemberGetter
 @Cast(value="const int64_t")
public static long kDefaultBatchSize()

\defgroup dataset-scanning Scanning API \{

kDefaultBatchReadahead

@Namespace(value="arrow::dataset")
 @MemberGetter
public static int kDefaultBatchReadahead()

kDefaultFragmentReadahead

@Namespace(value="arrow::dataset")
 @MemberGetter
public static int kDefaultFragmentReadahead()

kDefaultBackpressureHigh

@Namespace(value="arrow::dataset")
 @MemberGetter
public static int kDefaultBackpressureHigh()

kDefaultBackpressureLow

@Namespace(value="arrow::dataset")
 @MemberGetter
public static int kDefaultBackpressureLow()

InitializeScanner

@Namespace(value="arrow::dataset::internal")
public static void InitializeScanner(ExecFactoryRegistry registry)

\}

InitializeDatasetWriter

@Namespace(value="arrow::dataset::internal")
public static void InitializeDatasetWriter(ExecFactoryRegistry registry)

\}

kCsvTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static byte kCsvTypeName(int i)

kCsvTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static String kCsvTypeName()

kIpcTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static byte kIpcTypeName(int i)

\addtogroup dataset-file-formats \{

kIpcTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static String kIpcTypeName()

kParquetTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static byte kParquetTypeName(int i)

\addtogroup dataset-file-formats \{

kParquetTypeName

@Namespace(value="arrow::dataset")
 @MemberGetter
public static String kParquetTypeName()