@Override public void join(final Collection<FlowFile> parents, final FlowFile child, final String details, final long joinDuration) { verifyFlowFileKnown(child); try { final ProvenanceEventBuilder eventBuilder = build(child, ProvenanceEventType.JOIN); eventBuilder.addChildFlowFile(child); eventBuilder.setDetails(details); for (final FlowFile parent : parents) { eventBuilder.addParentFlowFile(parent); } events.add(eventBuilder.build()); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
private ProvenanceEventRecord createDropProvenanceEvent(final FlowFileRecord flowFile, final String requestor) { final ProvenanceEventBuilder builder = provRepository.eventBuilder(); builder.fromFlowFile(flowFile); builder.setEventType(ProvenanceEventType.DROP); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(getIdentifier()); builder.setComponentType("Connection"); builder.setAttributes(flowFile.getAttributes(), Collections.emptyMap()); builder.setDetails("FlowFile Queue emptied by " + requestor); builder.setSourceQueueIdentifier(getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize()); } return builder.build(); }
@Override public void associate(final FlowFile flowFile, final String alternateIdentifierNamespace, final String alternateIdentifier) { try { String trimmedNamespace = alternateIdentifierNamespace.trim(); if (trimmedNamespace.endsWith(":")) { trimmedNamespace = trimmedNamespace.substring(0, trimmedNamespace.length() - 1); } String trimmedIdentifier = alternateIdentifier.trim(); if (trimmedIdentifier.startsWith(":")) { if (trimmedIdentifier.length() == 1) { throw new IllegalArgumentException("Illegal alternateIdentifier: " + alternateIdentifier); } trimmedIdentifier = trimmedIdentifier.substring(1); } final String alternateIdentifierUri = trimmedNamespace + ":" + trimmedIdentifier; final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ADDINFO).setAlternateIdentifierUri(alternateIdentifierUri).build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
ProvenanceEventBuilder build(final FlowFile flowFile, final ProvenanceEventType eventType) { final ProvenanceEventBuilder builder = repository.eventBuilder(); builder.setEventType(eventType); builder.fromFlowFile(flowFile); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(processorId); builder.setComponentType(processorType); return builder; } }
/** * Generates a Fork event for the given child and parents but does not register the event. This is useful so that a ProcessSession has the ability to de-dupe events, since one or more events may * be created by the session itself, as well as by the Processor * * @param parents parents * @param child child * @return record */ ProvenanceEventRecord generateJoinEvent(final Collection<FlowFile> parents, final FlowFile child) { final ProvenanceEventBuilder eventBuilder = build(child, ProvenanceEventType.JOIN); eventBuilder.addChildFlowFile(child); for (final FlowFile parent : parents) { eventBuilder.addParentFlowFile(parent); } return eventBuilder.build(); }
ProvenanceEventRecord generateDropEvent(final FlowFile flowFile, final String details) { return build(flowFile, ProvenanceEventType.DROP).setDetails(details).build(); }
@Override public void receive(final FlowFile flowFile, final String transitUri, final String sourceSystemFlowFileIdentifier, final String details, final long transmissionMillis) { verifyFlowFileKnown(flowFile); try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.RECEIVE) .setTransitUri(transitUri).setSourceSystemFlowFileIdentifier(sourceSystemFlowFileIdentifier).setEventDuration(transmissionMillis).setDetails(details).build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String details) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.DROP) .setDetails(details) .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord dropEvent = builder.build(); return dropEvent; }
private void registerReceiveProvenanceEvents(final List<RemoteFlowFileRecord> flowFiles, final String nodeName, final String connectionId, final long startTimestamp) { final long duration = System.currentTimeMillis() - startTimestamp; final List<ProvenanceEventRecord> events = new ArrayList<>(flowFiles.size()); for (final RemoteFlowFileRecord remoteFlowFile : flowFiles) { final FlowFileRecord flowFileRecord = remoteFlowFile.getFlowFile(); final ProvenanceEventBuilder provenanceEventBuilder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFileRecord) .setEventType(ProvenanceEventType.RECEIVE) .setTransitUri("nifi://" + nodeName + "/loadbalance/" + connectionId) .setSourceSystemFlowFileIdentifier(remoteFlowFile.getRemoteUuid()) .setEventDuration(duration) .setComponentId(connectionId) .setComponentType("Load Balanced Connection"); final ContentClaim contentClaim = flowFileRecord.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); provenanceEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFileRecord.getContentClaimOffset(), flowFileRecord.getSize()); } final ProvenanceEventRecord provenanceEvent = provenanceEventBuilder.build(); events.add(provenanceEvent); } provenanceRepository.registerEvents(events); }
private void registerForkEvent(final FlowFile parent, final FlowFile child) { ProvenanceEventBuilder eventBuilder = forkEventBuilders.get(parent); if (eventBuilder == null) { eventBuilder = context.getProvenanceRepository().eventBuilder(); eventBuilder.setEventType(ProvenanceEventType.FORK); eventBuilder.setFlowFileEntryDate(parent.getEntryDate()); eventBuilder.setLineageStartDate(parent.getLineageStartDate()); eventBuilder.setFlowFileUUID(parent.getAttribute(CoreAttributes.UUID.key())); eventBuilder.setComponentId(context.getConnectable().getIdentifier()); final Connectable connectable = context.getConnectable(); final String processorType = connectable.getComponentType(); eventBuilder.setComponentType(processorType); eventBuilder.addParentFlowFile(parent); updateEventContentClaims(eventBuilder, parent, getRecord(parent)); forkEventBuilders.put(parent, eventBuilder); } eventBuilder.addChildFlowFile(child); }
.setEventType(ProvenanceEventType.EXPIRE) .setDetails("Expiration Threshold = " + getFlowFileExpiration()) .setComponentType("Load-Balanced Connection") .setComponentId(getIdentifier()) .setEventTime(System.currentTimeMillis()); builder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); final ProvenanceEventRecord provenanceEvent = builder.build(); provenanceEvents.add(provenanceEvent);
@Override public void route(final FlowFile flowFile, final Relationship relationship, final String details, final long processingDuration) { verifyFlowFileKnown(flowFile); try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ROUTE).setRelationship(relationship).setDetails(details).setEventDuration(processingDuration).build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
@Override public void invokeRemoteProcess(FlowFile flowFile, String transitUri, String details) { try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.REMOTE_INVOCATION) .setTransitUri(transitUri).setDetails(details).build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
@Override public void modifyContent(final FlowFile flowFile, final String details, final long processingMillis) { verifyFlowFileKnown(flowFile); try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.CONTENT_MODIFIED).setEventDuration(processingMillis).setDetails(details).build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
if (flowFiles.contains(eventFlowFile)) { final ProvenanceEventBuilder eventBuilder = entry.getValue(); for (final String childId : eventBuilder.getChildFlowFileIds()) { if (!flowFileIds.contains(childId)) { throw new IllegalStateException("Cannot migrate " + eventFlowFile + " to a new session because it was forked to create " + eventBuilder.getChildFlowFileIds().size() + " children and not all children are being migrated. If any FlowFile is forked, all of its children must also be migrated at the same time as the forked FlowFile"); final ProvenanceEventBuilder eventBuilder = entry.getValue(); final Set<String> childrenIds = new HashSet<>(eventBuilder.getChildFlowFileIds()); final String flowFileId = flowFile.getAttribute(CoreAttributes.UUID.key()); if (childrenIds.contains(flowFileId)) { eventBuilder.removeChildFlowFile(flowFile); copy = eventBuilder.copy(); copy.getChildFlowFileIds().clear(); copy.addChildFlowFile(flowFileId);
private void removeForkEvents(final FlowFile flowFile) { for (final ProvenanceEventBuilder builder : forkEventBuilders.values()) { final ProvenanceEventRecord event = builder.build(); if (event.getEventType() == ProvenanceEventType.FORK) { builder.removeChildFlowFile(flowFile); } } }
final ProvenanceEventRecord event = builder.build(); recordsToSubmit.add(provenanceReporter.build(curFlowFile, ProvenanceEventType.CONTENT_MODIFIED).build()); addEventType(eventTypesPerFlowFileId, flowFileId, ProvenanceEventType.CONTENT_MODIFIED); eventAdded = true; recordsToSubmit.add(provenanceReporter.build(curFlowFile, ProvenanceEventType.CREATE).build()); eventAdded = true; recordsToSubmit.add(provenanceReporter.build(curFlowFile, ProvenanceEventType.ATTRIBUTES_MODIFIED).build()); addEventType(eventTypesPerFlowFileId, flowFileId, ProvenanceEventType.ATTRIBUTES_MODIFIED);
@Override public void receive(final FlowFile flowFile, final String transitUri, final String sourceSystemFlowFileIdentifier, final String details, final long transmissionMillis) { verifyFlowFileKnown(flowFile); try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.RECEIVE) .setTransitUri(transitUri) .setSourceSystemFlowFileIdentifier(sourceSystemFlowFileIdentifier) .setEventDuration(transmissionMillis) .setDetails(details) .build(); events.add(record); } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
private ProvenanceEventRecord createSendEvent(final FlowFileRecord flowFile, final NodeIdentifier nodeIdentifier) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.SEND) .setDetails("Re-distributed for Load-balanced connection") .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()) .setSourceSystemFlowFileIdentifier(flowFile.getAttribute(CoreAttributes.UUID.key())) .setTransitUri("nifi://" + nodeIdentifier.getApiAddress() + "/loadbalance/" + flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord sendEvent = builder.build(); return sendEvent; }
ProvenanceEventBuilder build(final FlowFile flowFile, final ProvenanceEventType eventType) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventType(eventType); builder.fromFlowFile(flowFile); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(processorId); builder.setComponentType(processorType); return builder; }