org.opennms.nephron

Class Pipeline

java.lang.Object

org.opennms.nephron.Pipeline

public class Pipeline
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	Pipeline.CalculateFlowStatistics
static class	Pipeline.KeyFlowBy Maps flow documents into pairs of compound keys and aggregates.
static class	Pipeline.ReadFromKafka
static class	Pipeline.SumAndTopK
static class	Pipeline.SumsAndTopKs
static class	Pipeline.TotalAndSummary
static class	Pipeline.WindowedFlows
static class	Pipeline.WriteToElasticsearch
static class	Pipeline.WriteToKafka

Field Summary

Fields

Modifier and Type	Field and Description
static com.swrve.ratelimitedlogger.RateLimitedLog	RATE_LIMITED_LOG

Constructor Summary

Constructors

Constructor and Description
Pipeline()

Method Summary

Modifier and Type	Method and Description
static Pipeline.TotalAndSummary	aggregateParentTotal(String transformPrefix, org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<CompoundKey,Aggregate>> child)
static Pipeline.SumAndTopK	aggregateSumAndTopK(String transformPrefix, org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<CompoundKey,Aggregate>> groupedByKey, int k)
static Pipeline.SumsAndTopKs	aggregateSumsAndTopKs(String transformPrefix, org.apache.beam.sdk.values.PCollection<org.opennms.netmgt.flows.persistence.model.FlowDocument> input, CompoundKeyType typeWithTos, CompoundKeyType typeWithoutTos, int k)
static org.apache.beam.sdk.transforms.ParDo.SingleOutput<org.opennms.netmgt.flows.persistence.model.FlowDocument,org.opennms.netmgt.flows.persistence.model.FlowDocument>	attachTimestamps(org.joda.time.Duration fixedWindowSize, org.joda.time.Duration maxFlowDuration) Dispatches a FlowDocument to all of the windows that overlap with the flow range.
static org.apache.beam.sdk.Pipeline	create(NephronOptions options) Creates a new pipeline from the given set of runtime options.
static org.apache.beam.sdk.io.kafka.TimestampPolicyFactory<String,org.opennms.netmgt.flows.persistence.model.FlowDocument>	getKafkaInputTimestampPolicyFactory(org.joda.time.Duration maxDelay)
static void	registerCoders(org.apache.beam.sdk.Pipeline p)
static FlowSummary	toFlowSummary(FlowSummaryData fsd)
static FlowSummaryData	toFlowSummaryData(AggregationType aggregationType, org.apache.beam.sdk.transforms.windowing.IntervalWindow window, org.apache.beam.sdk.values.KV<CompoundKey,Aggregate> el, int ranking)
static org.apache.beam.sdk.transforms.windowing.Window<org.opennms.netmgt.flows.persistence.model.FlowDocument>	toWindow(org.joda.time.Duration fixedWindowSize, org.joda.time.Duration earlyProcessingDelay, org.joda.time.Duration lateProcessingDelay, org.joda.time.Duration allowedLateness)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

RATE_LIMITED_LOG

public static final com.swrve.ratelimitedlogger.RateLimitedLog RATE_LIMITED_LOG

Constructor Detail

Pipeline

public Pipeline()

Method Detail

create

public static org.apache.beam.sdk.Pipeline create(NephronOptions options)

Creates a new pipeline from the given set of runtime options.

Parameters:

options - runtime options

Returns:

a new pipeline

registerCoders

public static void registerCoders(org.apache.beam.sdk.Pipeline p)

getKafkaInputTimestampPolicyFactory

public static org.apache.beam.sdk.io.kafka.TimestampPolicyFactory<String,org.opennms.netmgt.flows.persistence.model.FlowDocument> getKafkaInputTimestampPolicyFactory(org.joda.time.Duration maxDelay)

attachTimestamps

public static org.apache.beam.sdk.transforms.ParDo.SingleOutput<org.opennms.netmgt.flows.persistence.model.FlowDocument,org.opennms.netmgt.flows.persistence.model.FlowDocument> attachTimestamps(org.joda.time.Duration fixedWindowSize,
                                                                                                                                                                                                  org.joda.time.Duration maxFlowDuration)

Dispatches a FlowDocument to all of the windows that overlap with the flow range.

Returns:

transform

toFlowSummaryData

public static FlowSummaryData toFlowSummaryData(AggregationType aggregationType,
                                                org.apache.beam.sdk.transforms.windowing.IntervalWindow window,
                                                org.apache.beam.sdk.values.KV<CompoundKey,Aggregate> el,
                                                int ranking)

toFlowSummary

public static FlowSummary toFlowSummary(FlowSummaryData fsd)

toWindow

public static org.apache.beam.sdk.transforms.windowing.Window<org.opennms.netmgt.flows.persistence.model.FlowDocument> toWindow(org.joda.time.Duration fixedWindowSize,
                                                                                                                                org.joda.time.Duration earlyProcessingDelay,
                                                                                                                                org.joda.time.Duration lateProcessingDelay,
                                                                                                                                org.joda.time.Duration allowedLateness)

aggregateParentTotal

public static Pipeline.TotalAndSummary aggregateParentTotal(String transformPrefix,
                                                            org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<CompoundKey,Aggregate>> child)

aggregateSumsAndTopKs

public static Pipeline.SumsAndTopKs aggregateSumsAndTopKs(String transformPrefix,
                                                          org.apache.beam.sdk.values.PCollection<org.opennms.netmgt.flows.persistence.model.FlowDocument> input,
                                                          CompoundKeyType typeWithTos,
                                                          CompoundKeyType typeWithoutTos,
                                                          int k)

aggregateSumAndTopK

public static Pipeline.SumAndTopK aggregateSumAndTopK(String transformPrefix,
                                                      org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<CompoundKey,Aggregate>> groupedByKey,
                                                      int k)