ProvenanceEventBuilder build(final FlowFile flowFile, final ProvenanceEventType eventType) { final ProvenanceEventBuilder builder = repository.eventBuilder(); builder.setEventType(eventType); builder.fromFlowFile(flowFile); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(processorId); builder.setComponentType(processorType); return builder; } }
@Override public void send(final FlowFile flowFile, final String transitUri, final String details, final long transmissionMillis, final boolean force) { try { final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.SEND).setTransitUri(transitUri).setEventDuration(transmissionMillis).setDetails(details).build(); // If the transmissionMillis field has been populated, use zero as the value of commitNanos (the call to System.nanoTime() is expensive but the value will be ignored). final long commitNanos = transmissionMillis < 0 ? System.nanoTime() : 0L; final ProvenanceEventRecord enriched = eventEnricher == null ? record : eventEnricher.enrich(record, flowFile, commitNanos); if (force) { repository.registerEvent(enriched); } else { events.add(enriched); } } catch (final Exception e) { logger.error("Failed to generate Provenance Event due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } } }
provRepo.registerEvents(provenanceEvents);
final StateManager stateManager = context.getStateManager(); Long currMaxId = eventAccess.getProvenanceRepository().getMaxEventId();
final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName(); Long currMaxId = context.getEventAccess().getProvenanceRepository().getMaxEventId();
protected QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException { // Create a Provenance Event and a FlowFile Repository record for each FlowFile final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size()); final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size()); for (final FlowFileRecord flowFile : flowFiles) { provenanceEvents.add(createDropProvenanceEvent(flowFile, requestor)); flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile)); } long dropContentSize = 0L; for (final FlowFileRecord flowFile : flowFiles) { dropContentSize += flowFile.getSize(); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim == null) { continue; } final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); if (resourceClaim == null) { continue; } resourceClaimManager.decrementClaimantCount(resourceClaim); } provRepository.registerEvents(provenanceEvents); flowFileRepository.updateRepository(flowFileRepoRecords); return new QueueSize(flowFiles.size(), dropContentSize); }
private void registerForkEvent(final FlowFile parent, final FlowFile child) { ProvenanceEventBuilder eventBuilder = forkEventBuilders.get(parent); if (eventBuilder == null) { eventBuilder = context.getProvenanceRepository().eventBuilder(); eventBuilder.setEventType(ProvenanceEventType.FORK); eventBuilder.setFlowFileEntryDate(parent.getEntryDate()); eventBuilder.setLineageStartDate(parent.getLineageStartDate()); eventBuilder.setFlowFileUUID(parent.getAttribute(CoreAttributes.UUID.key())); eventBuilder.setComponentId(context.getConnectable().getIdentifier()); final Connectable connectable = context.getConnectable(); final String processorType = connectable.getComponentType(); eventBuilder.setComponentType(processorType); eventBuilder.addParentFlowFile(parent); updateEventContentClaims(eventBuilder, parent, getRecord(parent)); forkEventBuilders.put(parent, eventBuilder); } eventBuilder.addChildFlowFile(child); }
private void handleContentNotFound(final ContentNotFoundException nfe, final StandardRepositoryRecord suspectRecord) { final ContentClaim registeredClaim = suspectRecord.getOriginalClaim(); final ContentClaim transientClaim = suspectRecord.getWorkingClaim(); final ContentClaim missingClaim = nfe.getMissingClaim(); final ProvenanceEventRecord dropEvent = provenanceReporter.drop(suspectRecord.getCurrent(), nfe.getMessage() == null ? "Content Not Found" : nfe.getMessage()); if (dropEvent != null) { context.getProvenanceRepository().registerEvent(dropEvent); } if (missingClaim == registeredClaim) { suspectRecord.markForAbort(); rollback(); throw new MissingFlowFileException("Unable to find content for FlowFile", nfe); } if (missingClaim == transientClaim) { rollback(); throw new MissingFlowFileException("Unable to find content for FlowFile", nfe); } }
provRepo.registerEvents(provenanceEvents);
private ProvenanceEventRecord createDropProvenanceEvent(final FlowFileRecord flowFile, final String requestor) { final ProvenanceEventBuilder builder = provRepository.eventBuilder(); builder.fromFlowFile(flowFile); builder.setEventType(ProvenanceEventType.DROP); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(getIdentifier()); builder.setComponentType("Connection"); builder.setAttributes(flowFile.getAttributes(), Collections.emptyMap()); builder.setDetails("FlowFile Queue emptied by " + requestor); builder.setSourceQueueIdentifier(getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize()); } return builder.build(); }
context.getProvenanceRepository().registerEvents(iterable); context.getFlowFileRepository().updateRepository(expiredRecords); } catch (final IOException e) {
provenanceRepo.registerEvents(iterable);