Package com.google.refine.importing

Class ImportingUtilities

java.lang.Object

com.google.refine.importing.ImportingUtilities

public class ImportingUtilities
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static interface	ImportingUtilities.Progress

Field Summary

Fields

Modifier and Type	Field	Description
static final List<String>	allowedProtocols
protected static final org.slf4j.Logger	logger

Constructor Summary

Constructors

Constructor	Description
ImportingUtilities()

Method Summary

Modifier and Type	Method	Description
static File	allocateFile(File dir, String name)
static String	autoSelectFiles(ImportingJob job, com.fasterxml.jackson.databind.node.ObjectNode retrievalRecord, com.fasterxml.jackson.databind.node.ArrayNode fileSelectionIndexes)	Figure out the best (most common) format for the set of files, select all files which match that format, and return the format found.
static long	createProject(ImportingJob job, String format, com.fasterxml.jackson.databind.node.ObjectNode optionObj, List<Exception> exceptions, boolean synchronous)
static ProjectMetadata	createProjectMetadata(com.fasterxml.jackson.databind.node.ObjectNode optionObj)
static String	getArchiveFileName(com.fasterxml.jackson.databind.node.ObjectNode fileRecord)
static String	getCommonFormatForSelectedFiles(ImportingJob job, com.fasterxml.jackson.databind.node.ArrayNode fileSelectionIndexes)
static String	getEncoding(com.fasterxml.jackson.databind.node.ObjectNode firstFileRecord)
static File	getFile(ImportingJob job, com.fasterxml.jackson.databind.node.ObjectNode fileRecord)
static File	getFile(ImportingJob job, String location)
static Reader	getFileReader(ImportingJob job, com.fasterxml.jackson.databind.node.ObjectNode fileRecord, String commonEncoding)
static Reader	getFileReader(File file, com.fasterxml.jackson.databind.node.ObjectNode fileRecord, String commonEncoding)
static String	getFileSource(com.fasterxml.jackson.databind.node.ObjectNode fileRecord)
static InputStreamReader	getInputStreamReader(InputStream is, String encoding)
static Reader	getReaderFromStream(InputStream inputStream, com.fasterxml.jackson.databind.node.ObjectNode fileRecord, String commonEncoding)
static String	getRelativePath(File file, File dir)
static boolean	hasArchiveFileField(List<com.fasterxml.jackson.databind.node.ObjectNode> fileRecords)
static boolean	isCompressed(File file)
static void	loadDataAndPrepareJob(javax.servlet.http.HttpServletRequest request, javax.servlet.http.HttpServletResponse response, Properties parameters, ImportingJob job, com.fasterxml.jackson.databind.node.ObjectNode config)
static String	normalizePath(String path)	Replace the illegal character with '-' in the path in Windows
static boolean	postProcessRetrievedFile(File rawDataDir, File file, com.fasterxml.jackson.databind.node.ObjectNode fileRecord, com.fasterxml.jackson.databind.node.ArrayNode fileRecords, ImportingUtilities.Progress progress)
static void	postProcessSingleRetrievedFile(File file, com.fasterxml.jackson.databind.node.ObjectNode fileRecord)
static void	previewParse(ImportingJob job, String format, com.fasterxml.jackson.databind.node.ObjectNode optionObj, List<Exception> exceptions)
static void	retrieveContentFromPostRequest(javax.servlet.http.HttpServletRequest request, Properties parameters, File rawDataDir, com.fasterxml.jackson.databind.node.ObjectNode retrievalRecord, ImportingUtilities.Progress progress)
static InputStream	tryOpenAsArchive(File file, String mimeType)
static InputStream	tryOpenAsArchive(File file, String mimeType, String contentType)
static InputStream	tryOpenAsCompressedFile(File file, String mimeType)
static InputStream	tryOpenAsCompressedFile(File file, String mimeType, String contentEncoding)
static File	uncompressFile(File rawDataDir, InputStream uncompressedIS, com.fasterxml.jackson.databind.node.ObjectNode fileRecord, ImportingUtilities.Progress progress)
static void	updateJobWithNewFileSelection(ImportingJob job, com.fasterxml.jackson.databind.node.ArrayNode fileSelectionArray)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

logger

protected static final org.slf4j.Logger logger

allowedProtocols

public static final List<String> allowedProtocols

Constructor Details

ImportingUtilities

public ImportingUtilities()

Method Details

loadDataAndPrepareJob

public static void loadDataAndPrepareJob(javax.servlet.http.HttpServletRequest request,
 javax.servlet.http.HttpServletResponse response,
 Properties parameters,
 ImportingJob job,
 com.fasterxml.jackson.databind.node.ObjectNode config)
                                  throws IOException,
javax.servlet.ServletException

Throws:

IOException

javax.servlet.ServletException

updateJobWithNewFileSelection

public static void updateJobWithNewFileSelection(ImportingJob job,
 com.fasterxml.jackson.databind.node.ArrayNode fileSelectionArray)

retrieveContentFromPostRequest

public static void retrieveContentFromPostRequest(javax.servlet.http.HttpServletRequest request,
 Properties parameters,
 File rawDataDir,
 com.fasterxml.jackson.databind.node.ObjectNode retrievalRecord,
 ImportingUtilities.Progress progress)
                                           throws IOException,
org.apache.commons.fileupload.FileUploadException

Throws:

IOException

org.apache.commons.fileupload.FileUploadException

getRelativePath

public static String getRelativePath(File file,
 File dir)

normalizePath

public static String normalizePath(String path)

Replace the illegal character with '-' in the path in Windows

Parameters:

path - : file path

Returns:

the replaced path or original path if the OS is not Windows

allocateFile

public static File allocateFile(File dir,
 String name)

getFileReader

public static Reader getFileReader(ImportingJob job,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord,
 String commonEncoding)
                            throws FileNotFoundException

Throws:

FileNotFoundException

getFileReader

public static Reader getFileReader(File file,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord,
 String commonEncoding)
                            throws FileNotFoundException

Throws:

FileNotFoundException

getReaderFromStream

public static Reader getReaderFromStream(InputStream inputStream,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord,
 String commonEncoding)

getFile

public static File getFile(ImportingJob job,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord)

getFile

public static File getFile(ImportingJob job,
 String location)

getFileSource

public static String getFileSource(com.fasterxml.jackson.databind.node.ObjectNode fileRecord)

getArchiveFileName

public static String getArchiveFileName(com.fasterxml.jackson.databind.node.ObjectNode fileRecord)

hasArchiveFileField

public static boolean hasArchiveFileField(List<com.fasterxml.jackson.databind.node.ObjectNode> fileRecords)

postProcessRetrievedFile

public static boolean postProcessRetrievedFile(File rawDataDir,
 File file,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord,
 com.fasterxml.jackson.databind.node.ArrayNode fileRecords,
 ImportingUtilities.Progress progress)
                                        throws IOException

Throws:

IOException

postProcessSingleRetrievedFile

public static void postProcessSingleRetrievedFile(File file,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord)

tryOpenAsArchive

public static InputStream tryOpenAsArchive(File file,
 String mimeType)
                                    throws IOException

Throws:

IOException

tryOpenAsArchive

public static InputStream tryOpenAsArchive(File file,
 String mimeType,
 String contentType)
                                    throws IOException

Throws:

IOException

isCompressed

public static boolean isCompressed(File file)
                            throws IOException

Throws:

IOException

tryOpenAsCompressedFile

public static InputStream tryOpenAsCompressedFile(File file,
 String mimeType)

tryOpenAsCompressedFile

public static InputStream tryOpenAsCompressedFile(File file,
 String mimeType,
 String contentEncoding)

uncompressFile

public static File uncompressFile(File rawDataDir,
 InputStream uncompressedIS,
 com.fasterxml.jackson.databind.node.ObjectNode fileRecord,
 ImportingUtilities.Progress progress)
                           throws IOException

Throws:

IOException

getEncoding

public static String getEncoding(com.fasterxml.jackson.databind.node.ObjectNode firstFileRecord)

autoSelectFiles

public static String autoSelectFiles(ImportingJob job,
 com.fasterxml.jackson.databind.node.ObjectNode retrievalRecord,
 com.fasterxml.jackson.databind.node.ArrayNode fileSelectionIndexes)

Figure out the best (most common) format for the set of files, select all files which match that format, and return the format found.

Parameters:

job - ImportingJob object

retrievalRecord - JSON object containing "files" key with all our files

fileSelectionIndexes - JSON array of selected file indices matching best format

Returns:

best (highest frequency) format

getCommonFormatForSelectedFiles

public static String getCommonFormatForSelectedFiles(ImportingJob job,
 com.fasterxml.jackson.databind.node.ArrayNode fileSelectionIndexes)

previewParse

public static void previewParse(ImportingJob job,
 String format,
 com.fasterxml.jackson.databind.node.ObjectNode optionObj,
 List<Exception> exceptions)

createProject

public static long createProject(ImportingJob job,
 String format,
 com.fasterxml.jackson.databind.node.ObjectNode optionObj,
 List<Exception> exceptions,
 boolean synchronous)

createProjectMetadata

public static ProjectMetadata createProjectMetadata(com.fasterxml.jackson.databind.node.ObjectNode optionObj)

getInputStreamReader

public static InputStreamReader getInputStreamReader(InputStream is,
 String encoding)
                                              throws IOException

Throws:

IOException