tl.lin.data.util

Class SequenceFileUtils

java.lang.Object

tl.lin.data.util.SequenceFileUtils

public class SequenceFileUtils
extends Object

Class containing a number of utility methods for manipulating SequenceFiles.

Method Summary

Methods

Modifier and Type	Method and Description
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readDirectory(org.apache.hadoop.fs.Path path)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readDirectory(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs, int max) Reads key-value pairs from a directory containing SequenceFiles.
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readFile(org.apache.hadoop.fs.Path path)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readFile(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readFile(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs, int max) Reads key-value pairs from a SequenceFile, up to a maximum number.
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>>	readFile(org.apache.hadoop.fs.Path path, int max)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V>	readFileIntoMap(org.apache.hadoop.fs.Path path)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V>	readFileIntoMap(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V>	readFileIntoMap(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs, int max)
static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V>	readFileIntoMap(org.apache.hadoop.fs.Path path, int max)
static <K extends org.apache.hadoop.io.Writable> List<K>	readKeys(org.apache.hadoop.fs.Path path)
static <K extends org.apache.hadoop.io.Writable> List<K>	readKeys(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs)
static <K extends org.apache.hadoop.io.Writable> List<K>	readKeys(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs, int max)
static <K extends org.apache.hadoop.io.Writable> List<K>	readKeys(org.apache.hadoop.fs.Path path, int max)
static <V extends org.apache.hadoop.io.Writable> List<V>	readValues(org.apache.hadoop.fs.Path path)
static <V extends org.apache.hadoop.io.Writable> List<V>	readValues(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs)
static <V extends org.apache.hadoop.io.Writable> List<V>	readValues(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileSystem fs, int max)
static <V extends org.apache.hadoop.io.Writable> List<V>	readValues(org.apache.hadoop.fs.Path path, int max)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

readFile

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readFile(org.apache.hadoop.fs.Path path)
                                                                                                                       throws IOException

Throws:

IOException

readFile

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readFile(org.apache.hadoop.fs.Path path,
                                                                                                                    int max)
                                                                                                                       throws IOException

Throws:

IOException

readFile

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readFile(org.apache.hadoop.fs.Path path,
                                                                                                                    org.apache.hadoop.fs.FileSystem fs)
                                                                                                                       throws IOException

Throws:

IOException

readFile

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readFile(org.apache.hadoop.fs.Path path,
                                                                                                                    org.apache.hadoop.fs.FileSystem fs,
                                                                                                                    int max)
                                                                                                                       throws IOException

Reads key-value pairs from a SequenceFile, up to a maximum number.

Parameters:

path - path to file

max - maximum of key-value pairs to read

Returns:

list of key-value pairs

Throws:

IOException

readFileIntoMap

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V> readFileIntoMap(org.apache.hadoop.fs.Path path)
                                                                                                                  throws IOException

Throws:

IOException

readFileIntoMap

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V> readFileIntoMap(org.apache.hadoop.fs.Path path,
                                                                                                               int max)
                                                                                                                  throws IOException

Throws:

IOException

readFileIntoMap

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V> readFileIntoMap(org.apache.hadoop.fs.Path path,
                                                                                                               org.apache.hadoop.fs.FileSystem fs)
                                                                                                                  throws IOException

Throws:

IOException

readFileIntoMap

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> SortedMap<K,V> readFileIntoMap(org.apache.hadoop.fs.Path path,
                                                                                                               org.apache.hadoop.fs.FileSystem fs,
                                                                                                               int max)
                                                                                                                  throws IOException

Throws:

IOException

readDirectory

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readDirectory(org.apache.hadoop.fs.Path path)

readDirectory

public static <K extends org.apache.hadoop.io.Writable,V extends org.apache.hadoop.io.Writable> List<PairOfWritables<K,V>> readDirectory(org.apache.hadoop.fs.Path path,
                                                                                                                         org.apache.hadoop.fs.FileSystem fs,
                                                                                                                         int max)

Reads key-value pairs from a directory containing SequenceFiles. A maximum number of key-value pairs is read from each SequenceFile.

Parameters:

path - path to directory

max - maximum of key-value pairs to read per file

Returns:

list of key-value pairs

readKeys

public static <K extends org.apache.hadoop.io.Writable> List<K> readKeys(org.apache.hadoop.fs.Path path)

readKeys

public static <K extends org.apache.hadoop.io.Writable> List<K> readKeys(org.apache.hadoop.fs.Path path,
                                                         int max)

readKeys

public static <K extends org.apache.hadoop.io.Writable> List<K> readKeys(org.apache.hadoop.fs.Path path,
                                                         org.apache.hadoop.fs.FileSystem fs)

readKeys

public static <K extends org.apache.hadoop.io.Writable> List<K> readKeys(org.apache.hadoop.fs.Path path,
                                                         org.apache.hadoop.fs.FileSystem fs,
                                                         int max)

readValues

public static <V extends org.apache.hadoop.io.Writable> List<V> readValues(org.apache.hadoop.fs.Path path)

readValues

public static <V extends org.apache.hadoop.io.Writable> List<V> readValues(org.apache.hadoop.fs.Path path,
                                                           int max)

readValues

public static <V extends org.apache.hadoop.io.Writable> List<V> readValues(org.apache.hadoop.fs.Path path,
                                                           org.apache.hadoop.fs.FileSystem fs)

readValues

public static <V extends org.apache.hadoop.io.Writable> List<V> readValues(org.apache.hadoop.fs.Path path,
                                                           org.apache.hadoop.fs.FileSystem fs,
                                                           int max)