Package org.deltafi.common.types

Class DeltaFile

java.lang.Object
org.deltafi.common.types.DeltaFile
public class DeltaFile extends Object

  • Field Details

    • CURRENT_SCHEMA_VERSION

      public static final int CURRENT_SCHEMA_VERSION
      See Also:

  • Constructor Details

    • DeltaFile

      public DeltaFile(DeltaFile other)

  • Method Details

    • snapshot

      public void snapshot()

    • getActions

      public List<Action> getActions()

    • getAnnotations

      public Map<String,String> getAnnotations()

    • getAnnotationKeys

      public Set<String> getAnnotationKeys()

    • setStage

      public void setStage(DeltaFileStage stage)

    • updateFlags

      public void updateFlags()

    • setContentDeleted

      public void setContentDeleted(OffsetDateTime contentDeleted)

    • getMetadata

      public Map<String,String> getMetadata()
      Get the cumulative metadata from all actions that amend data through the load and enrich actions as well as retried format actions This method retrieves a copy of source metadata, and then applies changes from all actions that amended data. For each such action, the method adds its metadata to the Map. If any metadata key is present in the action's deleteMetadataKeys, the corresponding entry is removed from the Map.
      Returns:
      A Map containing the resulting metadata

    • erroredActions

      public List<Action> erroredActions()

    • queueAction

      public Action queueAction(String flow, String name, ActionType type, boolean coldQueue)

    • queueNewAction

      public Action queueNewAction(String flow, String name, ActionType type, boolean coldQueue)

    • addAction

      public Action addAction(String flow, String name, ActionType type, ActionState state)

    • actionNamed

      public Optional<Action> actionNamed(String flow, String name)

    • firstActionError

      public Optional<Action> firstActionError()

    • isNewAction

      public boolean isNewAction(String flow, String name)

    • completeAction

      public void completeAction(ActionEvent event)

    • completeAction

      public void completeAction(ActionEvent event, List<Content> content, Map<String,String> metadata, List<String> deleteMetadataKeys)

    • completeAction

      public Action completeAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop)

    • completeAction

      public Action completeAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, List<Content> content, Map<String,String> metadata, List<String> deleteMetadataKeys)

    • split

      public Action split(ActionEvent event, List<String> childDids)

    • collectedAction

      public void collectedAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop)

    • filterAction

      public void filterAction(ActionEvent event)

    • lastAction

      public Action lastAction()

    • errorAction

      public void errorAction(ActionEvent event)

    • errorAction

      public void errorAction(ActionEvent event, String policyName, Integer delay)

    • errorAction

      public void errorAction(ActionEvent event, String errorCause, String errorContext)

    • errorAction

      public void errorAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, String errorCause, String errorContext)

    • errorAction

      public void errorAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, String errorCause, String errorContext, OffsetDateTime nextAutoResume)

    • retryErrors

      public List<String> retryErrors(@NotNull List<ResumeMetadata> resumeMetadata)

    • setTestModeReason

      public void setTestModeReason(String reason)

    • queuedActions

      public List<String> queuedActions()

    • addAnnotations

      public void addAnnotations(Map<String,String> metadata)

    • addAnnotationsIfAbsent

      public void addAnnotationsIfAbsent(Map<String,String> metadata)

    • addAnnotationIfAbsent

      public void addAnnotationIfAbsent(String key, String value)

    • addEgressFlow

      public void addEgressFlow(@NotNull String flow)

    • hasErroredAction

      public boolean hasErroredAction()

    • hasPendingActions

      public boolean hasPendingActions()

    • hasFilteredAction

      public boolean hasFilteredAction()

    • hasCollectingAction

      public boolean hasCollectingAction()

    • hasActionInState

      public boolean hasActionInState(ActionState actionState)

    • ensurePendingAction

      public void ensurePendingAction(String flow, String name)

    • hasTerminalAction

      public boolean hasTerminalAction(String flow, String name)

    • hasCompletedAction

      public boolean hasCompletedAction(String flow, String name)

    • hasCompletedActions

      public boolean hasCompletedActions(String flow, List<String> names)

    • hasCollectedAction

      public boolean hasCollectedAction(String flow, String name)

    • hasActionInState

      public boolean hasActionInState(String flow, String name, ActionState state)

    • sourceMetadata

      public String sourceMetadata(String key)

    • sourceMetadata

      public String sourceMetadata(String key, String defaultValue)

    • addPendingAnnotationsForFlow

      public void addPendingAnnotationsForFlow(String flowName)
      Add the given flow to the set of pendingAnnotationsForFlows for this DeltaFile Do nothing if the given set is null or empty
      Parameters:
      flowName - name of the flow that requires annotations for this DeltaFile

    • updatePendingAnnotationsForFlows

      public void updatePendingAnnotationsForFlows(String flow, Set<String> expectedAnnotations)
      Check if the set of expected annotations are satisfied for the given flow. If the all the annotations are present, remove the flow from the pendingAnnotationsForFlows set. If pendingAnnotationsForFlows is empty after removing the flow, set it to null.
      Parameters:
      flow - name of the flow that could be removed from pendingAnnotationsForFlows
      expectedAnnotations - set of annotations expected for the given flow

    • pendingAnnotations

      public Set<String> pendingAnnotations(Set<String> expectedAnnotations)
      Get the set of annotations that are still pending from the given set of expected annotations
      Parameters:
      expectedAnnotations - annotations that are expected to be set on the DeltaFile
      Returns:
      annotations that have not been added to the DeltaFile yet

    • lastCompleteAction

      public Action lastCompleteAction()

    • lastContent

      @NotNull public List<Content> lastContent()

    • getEgress

      @NotNull public List<Egress> getEgress()

    • forQueue

      public DeltaFileMessage forQueue()

    • referencedSegments

      public Set<Segment> referencedSegments()

    • storedSegments

      public Set<Segment> storedSegments()

    • recalculateBytes

      public void recalculateBytes()

    • cancel

      public void cancel()

    • cancelQueuedActions

      public void cancelQueuedActions()

    • incrementRequeueCount

      public void incrementRequeueCount()

    • canBeCancelled

      public boolean canBeCancelled()

    • inactiveStage

      public boolean inactiveStage()

    • lastAction

      public Action lastAction(String actionName)

    • acknowledgeError

      public void acknowledgeError(@NotNull OffsetDateTime now, String reason)

    • clearErrorAcknowledged

      public void clearErrorAcknowledged()

    • generateUpdate

      public org.springframework.data.mongodb.core.query.Update generateUpdate()

    • getImmutableMetadata

      public Map<String,String> getImmutableMetadata()

    • getImmutableContent

      public List<Content> getImmutableContent()