Package org.deltafi.common.types

Class DeltaFile

java.lang.Object
org.deltafi.common.types.DeltaFile
public class DeltaFile extends Object

  • Field Details

    • CURRENT_SCHEMA_VERSION

      public static final int CURRENT_SCHEMA_VERSION
      See Also:

  • Constructor Details

    • DeltaFile

      public DeltaFile(DeltaFile other)

  • Method Details

    • snapshot

      public void snapshot()

    • getActions

      public List<Action> getActions()

    • getAnnotations

      public Map<String,String> getAnnotations()

    • getAnnotationKeys

      public Set<String> getAnnotationKeys()

    • setStage

      public void setStage(DeltaFileStage stage)

    • updateFlags

      public void updateFlags()

    • setContentDeleted

      public void setContentDeleted(OffsetDateTime contentDeleted)

    • getMetadata

      public Map<String,String> getMetadata()
      Get the cumulative metadata from all actions that amend data through the load and enrich actions as well as retried format actions This method retrieves a copy of source metadata, and then applies changes from all actions that amended data. For each such action, the method adds its metadata to the Map. If any metadata key is present in the action's deleteMetadataKeys, the corresponding entry is removed from the Map.
      Returns:
      A Map containing the resulting metadata

    • metadataWithFormatRetries

      public Map<String,String> metadataWithFormatRetries(String flow)

    • getErrorMetadata

      public Map<String,String> getErrorMetadata(Action action)
      Get metadata, considering whether the given action is post-formatting. This method checks if the provided action follows a format action. If it does, the method retrieves formatted metadata for the action's associated egress flow. Otherwise, it retrieves the current ingress/transform flow metadata.
      Parameters:
      action - The action based on which the decision to format or not is made.
      Returns:
      A Map containing either the formatted metadata for the action's flow, or the current transform/load metadata.

    • erroredActions

      public List<Action> erroredActions()

    • queueAction

      public Action queueAction(String flow, String name, ActionType type, boolean coldQueue)

    • queueNewAction

      public Action queueNewAction(String flow, String name, ActionType type, boolean coldQueue)

    • addAction

      public Action addAction(String flow, String name, ActionType type, ActionState state)

    • addReadyToCollectAction

      public void addReadyToCollectAction(String flow, String name, ActionType type)

    • actionNamed

      public Optional<Action> actionNamed(String flow, String name)

    • firstActionError

      public Optional<Action> firstActionError()

    • isNewAction

      public boolean isNewAction(String flow, String name)

    • completeAction

      public Action completeAction(ActionEvent event)

    • completeAction

      public void completeAction(ActionEvent event, List<Content> content, Map<String,String> metadata, List<String> deleteMetadataKeys, List<Domain> domains, List<Enrichment> enrichments)

    • completeAction

      public Action completeAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop)

    • completeAction

      public Action completeAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, List<Content> content, Map<String,String> metadata, List<String> deleteMetadataKeys, List<Domain> domains, List<Enrichment> enrichments)

    • collectingAction

      public void collectingAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop)

    • collectedAction

      public void collectedAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop)

    • filterAction

      public void filterAction(ActionEvent event, String filterMessage)

    • reinjectAction

      public void reinjectAction(ActionEvent event)

    • lastAction

      public Action lastAction()

    • setLastActionReinjected

      public void setLastActionReinjected()

    • removeLastAction

      public void removeLastAction()

    • errorAction

      public void errorAction(ActionEvent event)

    • errorAction

      public void errorAction(ActionEvent event, String policyName, Integer delay)

    • errorAction

      public void errorAction(ActionEvent event, String errorCause, String errorContext)

    • errorAction

      public void errorAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, String errorCause, String errorContext)

    • errorAction

      public void errorAction(String flow, String name, OffsetDateTime start, OffsetDateTime stop, String errorCause, String errorContext, OffsetDateTime nextAutoResume)

    • retryErrors

      public List<String> retryErrors(@NotNull List<ResumeMetadata> resumeMetadata)

    • setTestModeReason

      public void setTestModeReason(String reason)

    • queuedActions

      public List<String> queuedActions()

    • readyToCollectActions

      public List<Action> readyToCollectActions()

    • hasDomains

      public boolean hasDomains(List<String> domains)

    • addAnnotations

      public void addAnnotations(Map<String,String> metadata)

    • addAnnotationsIfAbsent

      public void addAnnotationsIfAbsent(Map<String,String> metadata)

    • addAnnotationIfAbsent

      public void addAnnotationIfAbsent(String key, String value)

    • addEgressFlow

      public void addEgressFlow(@NotNull String flow)

    • hasEnrichments

      public boolean hasEnrichments(List<String> enrichments)

    • hasErroredAction

      public boolean hasErroredAction()

    • hasPendingActions

      public boolean hasPendingActions()

    • hasFilteredAction

      public boolean hasFilteredAction()

    • hasReinjectedAction

      public boolean hasReinjectedAction()

    • hasActionInState

      public boolean hasActionInState(ActionState actionState)

    • ensurePendingAction

      public void ensurePendingAction(String flow, String name)

    • getPendingAction

      public Action getPendingAction(String flow, String name)

    • hasTerminalAction

      public boolean hasTerminalAction(String flow, String name)

    • hasCompletedAction

      public boolean hasCompletedAction(String flow, String name)

    • hasCompletedActions

      public boolean hasCompletedActions(String flow, List<String> names)

    • hasCollectedAction

      public boolean hasCollectedAction(String flow, String name)

    • hasActionInState

      public boolean hasActionInState(String flow, String name, ActionState state)

    • sourceMetadata

      public String sourceMetadata(String key)

    • sourceMetadata

      public String sourceMetadata(String key, String defaultValue)

    • addPendingAnnotationsForFlow

      public void addPendingAnnotationsForFlow(String flowName)
      Add the given flow to the set of pendingAnnotationsForFlows for this DeltaFile Do nothing if the given set is null or empty
      Parameters:
      flowName - name of the flow that requires annotations for this DeltaFile

    • updatePendingAnnotationsForFlows

      public void updatePendingAnnotationsForFlows(String flow, Set<String> expectedAnnotations)
      Check if the set of expected annotations are satisfied for the given flow. If the all the annotations are present, remove the flow from the pendingAnnotationsForFlows set. If pendingAnnotationsForFlows is empty after removing the flow, set it to null.
      Parameters:
      flow - name of the flow that could be removed from pendingAnnotationsForFlows
      expectedAnnotations - set of annotations expected for the given flow

    • pendingAnnotations

      public Set<String> pendingAnnotations(Set<String> expectedAnnotations)
      Get the set of annotations that are still pending from the given set of expected annotations
      Parameters:
      expectedAnnotations - annotations that are expected to be set on the DeltaFile
      Returns:
      annotations that have not been added to the DeltaFile yet

    • lastCompleteDataAmendedAction

      public Action lastCompleteDataAmendedAction()

    • lastDataAmendedContent

      @NotNull public List<Content> lastDataAmendedContent()

    • domains

      @NotNull public List<Domain> domains()

    • enrichments

      @NotNull public List<Enrichment> enrichments()

    • getEgress

      @NotNull public List<Egress> getEgress()

    • lastFormatAction

      public Action lastFormatAction(String flow)

    • retriedEgressActions

      public List<Action> retriedEgressActions(String flow)

    • formatComplete

      public boolean formatComplete(String flow)

    • formatMetadata

      public Map<String,String> formatMetadata(String flow)
      Assemble formatted metadata based on a given flow. This method prepares metadata by iterating through the format actions and retried egress actions associated with the provided flow. The metadata from each format action is added to a Map. If any metadata key is present in the action's deleteMetadataKeys, the corresponding entry is removed from the Map.
      Parameters:
      flow - The flow containing the format actions
      Returns:
      A Map containing the formatted metadata

    • formatContent

      public List<Content> formatContent(String flow)

    • forQueue

      public DeltaFileMessage forQueue(String flow)

    • referencedSegments

      public Set<Segment> referencedSegments()

    • storedSegments

      public Set<Segment> storedSegments()

    • recalculateBytes

      public void recalculateBytes()

    • cancel

      public void cancel()

    • cancelQueuedActions

      public void cancelQueuedActions()

    • incrementRequeueCount

      public void incrementRequeueCount()

    • canBeCancelled

      public boolean canBeCancelled()

    • inactiveStage

      public boolean inactiveStage()

    • lastAction

      public Action lastAction(String actionName)

    • acknowledgeError

      public void acknowledgeError(@NotNull OffsetDateTime now, String reason)

    • clearErrorAcknowledged

      public void clearErrorAcknowledged()

    • generateUpdate

      public org.springframework.data.mongodb.core.query.Update generateUpdate()