org.lockss.laaws.rs.io.storage.hdfs

Class WARCFileWriter

java.lang.Object

org.springframework.data.hadoop.store.support.LifecycleObjectSupport

org.springframework.data.hadoop.store.support.StoreObjectSupport

org.springframework.data.hadoop.store.support.OutputStoreObjectSupport

org.springframework.data.hadoop.store.output.AbstractDataStreamWriter

org.lockss.laaws.rs.io.storage.hdfs.WARCFileWriter

All Implemented Interfaces:

Closeable, Flushable, AutoCloseable, org.archive.format.ArchiveFileConstants, org.archive.format.warc.WARCConstants, org.springframework.beans.factory.Aware, org.springframework.beans.factory.BeanFactoryAware, org.springframework.beans.factory.InitializingBean, org.springframework.context.Lifecycle, org.springframework.context.Phased, org.springframework.context.SmartLifecycle, org.springframework.data.hadoop.store.DataStoreWriter<org.archive.io.warc.WARCRecordInfo>, org.springframework.data.hadoop.store.DataWriter<org.archive.io.warc.WARCRecordInfo>

public class WARCFileWriter
extends org.springframework.data.hadoop.store.output.AbstractDataStreamWriter
implements org.springframework.data.hadoop.store.DataStoreWriter<org.archive.io.warc.WARCRecordInfo>, org.archive.format.warc.WARCConstants

Nested Class Summary

Nested classes/interfaces inherited from interface org.archive.format.warc.WARCConstants

org.archive.format.warc.WARCConstants.WARCRecordType

Field Summary

Fields inherited from class org.springframework.data.hadoop.store.output.AbstractDataStreamWriter

DEFAULT_MAX_OPEN_ATTEMPTS

Fields inherited from interface org.archive.format.warc.WARCConstants

COLON_SPACE, COMPRESSED_WARC_FILE_EXTENSION, CONTENT_DESCRIPTION, CONTENT_LENGTH, CONTENT_TYPE, DEFAULT_ENCODING, DEFAULT_MAX_WARC_FILE_SIZE, DOT_COMPRESSED_WARC_FILE_EXTENSION, DOT_WARC_FILE_EXTENSION, FTP_CONTROL_CONVERSATION_MIMETYPE, HEADER_FIELD_SEPARATOR, HEADER_KEY_BLOCK_DIGEST, HEADER_KEY_CONCURRENT_TO, HEADER_KEY_DATE, HEADER_KEY_ETAG, HEADER_KEY_FILENAME, HEADER_KEY_ID, HEADER_KEY_IP, HEADER_KEY_LAST_MODIFIED, HEADER_KEY_PAYLOAD_DIGEST, HEADER_KEY_PROFILE, HEADER_KEY_REFERS_TO, HEADER_KEY_REFERS_TO_DATE, HEADER_KEY_REFERS_TO_FILE_OFFSET, HEADER_KEY_REFERS_TO_FILENAME, HEADER_KEY_REFERS_TO_TARGET_URI, HEADER_KEY_TRUNCATED, HEADER_KEY_TYPE, HEADER_KEY_URI, HEADER_LINE_ENCODING, HTTP_REQUEST_MIMETYPE, HTTP_RESPONSE_MIMETYPE, MAX_LINE_LENGTH, MAX_WARC_HEADER_LINE_LENGTH, NAMED_FIELD_CHECKSUM_LABEL, NAMED_FIELD_DESCRIPTION, NAMED_FIELD_FILEDESC, NAMED_FIELD_IP_LABEL, NAMED_FIELD_RELATED_LABEL, NAMED_FIELD_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_HEAD, NAMED_FIELD_TRUNCATED_VALUE_LENGTH, NAMED_FIELD_TRUNCATED_VALUE_TIME, NAMED_FIELD_TRUNCATED_VALUE_UNSPECIFIED, NAMED_FIELD_WARCFILENAME, PLACEHOLDER_RECORD_LENGTH_STRING, PROFILE_REVISIT_IDENTICAL_DIGEST, PROFILE_REVISIT_NOT_MODIFIED, TRUNCATED_VALUE_UNSPECIFIED, TYPE, WARC_FIELDS_TYPE, WARC_FILE_EXTENSION, WARC_HEADER_ENCODING, WARC_ID, WARC_MAGIC, WARC_VERSION, WSP

Fields inherited from interface org.archive.format.ArchiveFileConstants

ABSOLUTE_OFFSET_KEY, CDX, CDX_FILE, CDX_LINE_BUFFER_SIZE, DATE_FIELD_KEY, DEFAULT_DIGEST_METHOD, DOT_COMPRESSED_FILE_EXTENSION, DUMP, GZIP_DUMP, HEADER, INVALID_SUFFIX, LENGTH_FIELD_KEY, MIMETYPE_FIELD_KEY, NOHEAD, OCCUPIED_SUFFIX, ORIGIN_FIELD_KEY, READER_IDENTIFIER_FIELD_KEY, RECORD_IDENTIFIER_FIELD_KEY, SINGLE_SPACE, TYPE_FIELD_KEY, URL_FIELD_KEY, VERSION_FIELD_KEY

Constructor Summary

Constructors

Constructor and Description
WARCFileWriter(org.apache.hadoop.conf.Configuration configuration, org.apache.hadoop.fs.Path basePath, org.springframework.data.hadoop.store.codec.CodecInfo codec)

Method Summary

Modifier and Type	Method and Description
void	close()
protected static String	createRecordHeader(org.archive.io.warc.WARCRecordInfo record)
org.archive.io.warc.WARCRecordInfo	createWARCInfoRecord()
void	flush()
org.springframework.data.hadoop.store.support.StreamsHolder<OutputStream>	getOutput()
long	getPosition()
protected void	handleTimeout()
void	hflush()
void	write(org.archive.io.warc.WARCRecordInfo record)
static void	write(org.archive.io.warc.WARCRecordInfo record, OutputStream out)

Methods inherited from class org.springframework.data.hadoop.store.output.AbstractDataStreamWriter

getPosition, setMaxOpenAttempts

Methods inherited from class org.springframework.data.hadoop.store.support.OutputStoreObjectSupport

findInitFiles, getOutputContext, getResolvedPath, initOutputContext, isAppendable, isFileWriteable, isOverwrite, isSyncable, onInit, renameFile, setAppendable, setFileNamingStrategy, setInWritingPrefix, setInWritingSuffix, setOverwrite, setRolloverStrategy, setSyncable, setWritePosition

Methods inherited from class org.springframework.data.hadoop.store.support.StoreObjectSupport

doStart, doStop, flushTimeout, getCodec, getConfiguration, getPath, isCompressed, resetIdleTimeout, setCloseTimeout, setFlushTimeout, setIdleTimeout

Methods inherited from class org.springframework.data.hadoop.store.support.LifecycleObjectSupport

afterPropertiesSet, getBeanFactory, getPhase, getStoreEventPublisher, getTaskExecutor, getTaskScheduler, isAutoStartup, isRunning, setAutoStartup, setBeanFactory, setPhase, setStoreEventPublisher, setTaskExecutor, setTaskScheduler, start, stop, stop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

WARCFileWriter

public WARCFileWriter(org.apache.hadoop.conf.Configuration configuration,
                      org.apache.hadoop.fs.Path basePath,
                      org.springframework.data.hadoop.store.codec.CodecInfo codec)

Method Detail

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Throws:

IOException

flush

public void flush()
           throws IOException

Specified by:

flush in interface Flushable

Throws:

IOException

hflush

public void hflush()
            throws IOException

Throws:

IOException

handleTimeout

protected void handleTimeout()

Overrides:

handleTimeout in class org.springframework.data.hadoop.store.support.StoreObjectSupport

getOutput

public org.springframework.data.hadoop.store.support.StreamsHolder<OutputStream> getOutput()
                                                                                    throws IOException

Overrides:

getOutput in class org.springframework.data.hadoop.store.output.AbstractDataStreamWriter

Throws:

IOException

createWARCInfoRecord

public org.archive.io.warc.WARCRecordInfo createWARCInfoRecord()
                                                        throws IOException

Throws:

IOException

write

public void write(org.archive.io.warc.WARCRecordInfo record)
           throws IOException

Specified by:

write in interface org.springframework.data.hadoop.store.DataWriter<org.archive.io.warc.WARCRecordInfo>

Throws:

IOException

getPosition

public long getPosition()
                 throws IOException

Throws:

IOException

write

public static void write(org.archive.io.warc.WARCRecordInfo record,
                         OutputStream out)
                  throws IOException

Throws:

IOException

createRecordHeader

protected static String createRecordHeader(org.archive.io.warc.WARCRecordInfo record)