final ProvenanceEventRecord replayEvent = new StandardProvenanceEventRecord.Builder() .setEventType(ProvenanceEventType.REPLAY) .addChildUuid(newFlowFileUUID) .addParentUuid(parentUUID) .setFlowFileUUID(parentUUID) .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes()) .setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize()) .setDetails("Replay requested by " + user.getIdentity()) .setEventTime(System.currentTimeMillis()) .setFlowFileEntryDate(System.currentTimeMillis()) .setLineageStartDate(event.getLineageStartDate()) .setComponentType(event.getComponentType()) .setComponentId(event.getComponentId()) .build(); provenanceRepository.registerEvent(replayEvent);
private StandardProvenanceEventRecord truncateAttributes(final StandardProvenanceEventRecord original) { boolean requireTruncation = false; for (final String updatedAttr : original.getUpdatedAttributes().values()) { if (updatedAttr != null && updatedAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } if (!requireTruncation) { for (final String previousAttr : original.getPreviousAttributes().values()) { if (previousAttr != null && previousAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } } if (!requireTruncation) { return original; } final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder().fromEvent(original); builder.setAttributes(truncateAttributes(original.getPreviousAttributes()), truncateAttributes(original.getUpdatedAttributes())); final StandardProvenanceEventRecord truncated = builder.build(); truncated.setEventId(original.getEventId()); return truncated; }
@Override public ProvenanceEventBuilder copy() { final Builder copy = new Builder(); copy.eventTime = eventTime; copy.entryDate = entryDate;
private StandardProvenanceEventRecord truncateAttributes(final StandardProvenanceEventRecord original) { boolean requireTruncation = false; for (final String updatedAttr : original.getUpdatedAttributes().values()) { if (updatedAttr != null && updatedAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } if (!requireTruncation) { for (final String previousAttr : original.getPreviousAttributes().values()) { if (previousAttr != null && previousAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } } if (!requireTruncation) { return original; } final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder().fromEvent(original); builder.setAttributes(truncateAttributes(original.getPreviousAttributes()), truncateAttributes(original.getUpdatedAttributes())); final StandardProvenanceEventRecord truncated = builder.build(); truncated.setEventId(original.getEventId()); return truncated; }
@Override public StandardProvenanceEventRecord.Builder eventBuilder() { return new StandardProvenanceEventRecord.Builder(); }
@Override public ProvenanceEventBuilder eventBuilder() { return new StandardProvenanceEventRecord.Builder(); }
ProvenanceEventBuilder build(final FlowFile flowFile, final ProvenanceEventType eventType) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventType(eventType); builder.fromFlowFile(flowFile); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(processorId); builder.setComponentType(processorType); return builder; }
@Override public ProvenanceEventBuilder eventBuilder() { return new StandardProvenanceEventRecord.Builder(); }
@Override public ProvenanceEventBuilder eventBuilder() { return new StandardProvenanceEventRecord.Builder(); }
@Override public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) { setFlowFileEntryDate(flowFile.getEntryDate()); setLineageStartDate(flowFile.getLineageStartDate()); setAttributes(Collections.emptyMap(), flowFile.getAttributes()); uuid = flowFile.getAttribute(CoreAttributes.UUID.key()); this.contentSize = flowFile.getSize(); return this; }
@Override public void registerEvent(final ProvenanceEventRecord event) { final StandardProvenanceEventRecord newRecord; if (event instanceof StandardProvenanceEventRecord) { newRecord = (StandardProvenanceEventRecord) event; } else { newRecord = new StandardProvenanceEventRecord.Builder().fromEvent(event).build(); } newRecord.setEventId(idGenerator.getAndIncrement()); records.add(newRecord); }
final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder(); builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER)); builder.setChildUuids(childUuids); builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS)); builder.setParentUuids(parentUuids); builder.setPreviousAttributes(previousAttributes); builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP)); builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER)); builder.setTransitUri((String) record.getFieldValue(EventFieldNames.TRANSIT_URI)); builder.setUpdatedAttributes(updatedAttributes); builder.setComponentId(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_ID), componentIds)); builder.setComponentType(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_TYPE), componentTypes)); builder.setSourceQueueIdentifier(readLookupValue(record.getFieldValue(EventFieldNames.SOURCE_QUEUE_IDENTIFIER), queueIds)); builder.setEventType(eventType); builder.setFlowFileUUID(uuid); builder.setEventDuration((Integer) record.getFieldValue(EventFieldNames.EVENT_DURATION)); builder.setEventTime(addLong((Integer) record.getFieldValue(EventFieldNames.EVENT_TIME), startTimeOffset)); builder.setFlowFileEntryDate(addLong((Integer) record.getFieldValue(EventFieldNames.FLOWFILE_ENTRY_DATE), startTimeOffset)); builder.setLineageStartDate(addLong((Integer) record.getFieldValue(EventFieldNames.LINEAGE_START_DATE), startTimeOffset)); builder.setEventId(eventId.longValue() + eventIdStartOffset); builder.setStorageLocation(storageFilename, storageByteOffset); builder.setPreviousContentClaim(
final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventType(eventType); builder.setEventTime(dis.readLong()); builder.setEventDuration(dis.readLong()); builder.setCurrentContentClaim(null, null, null, null, fileSize); builder.setComponentId(readNullableString(dis)); builder.setComponentType(readNullableString(dis)); builder.setFlowFileUUID(uuid); builder.setDetails(readNullableString(dis)); final Map<String, String> previousAttrs = readAttributes(dis, false); final Map<String, String> attrUpdates = readAttributes(dis, true); builder.setAttributes(previousAttrs, attrUpdates); builder.setCurrentContentClaim(dis.readUTF(), dis.readUTF(), dis.readUTF(), dis.readLong(), dis.readLong()); } else { builder.setCurrentContentClaim(null, null, null, null, 0L); builder.setPreviousContentClaim(dis.readUTF(), dis.readUTF(), dis.readUTF(), dis.readLong(), dis.readLong()); builder.setSourceQueueIdentifier(readNullableString(dis)); } else { final Map<String, String> attrs = readAttributes(dis, false); builder.setAttributes(Collections.<String, String>emptyMap(), attrs);
expiredRecords.add(record); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.EXPIRE) .setDetails("Expiration Threshold = " + getFlowFileExpiration())
@SuppressWarnings("unchecked") public static StandardProvenanceEventRecord getEvent(final Record record, final String storageFilename, final long storageByteOffset, final int maxAttributeLength) { final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder(); builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER)); builder.setChildUuids((List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS)); builder.setComponentId((String) record.getFieldValue(EventFieldNames.COMPONENT_ID)); builder.setComponentType((String) record.getFieldValue(EventFieldNames.COMPONENT_TYPE)); builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS)); builder.setEventDuration((Long) record.getFieldValue(EventFieldNames.EVENT_DURATION)); builder.setEventTime((Long) record.getFieldValue(EventFieldNames.EVENT_TIME)); builder.setEventType(ProvenanceEventType.valueOf((String) record.getFieldValue(EventFieldNames.EVENT_TYPE))); builder.setFlowFileEntryDate((Long) record.getFieldValue(EventFieldNames.FLOWFILE_ENTRY_DATE)); builder.setFlowFileUUID((String) record.getFieldValue(EventFieldNames.FLOWFILE_UUID)); builder.setLineageStartDate((Long) record.getFieldValue(EventFieldNames.LINEAGE_START_DATE)); builder.setParentUuids((List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS)); builder.setPreviousAttributes(truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.PREVIOUS_ATTRIBUTES), maxAttributeLength)); builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP)); builder.setSourceQueueIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_QUEUE_IDENTIFIER)); builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER)); builder.setTransitUri((String) record.getFieldValue(EventFieldNames.TRANSIT_URI)); builder.setUpdatedAttributes(truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.UPDATED_ATTRIBUTES), maxAttributeLength)); builder.setEventId(eventId); builder.setStorageLocation(storageFilename, storageByteOffset); builder.setCurrentContentClaim(null, null, null, null, 0L); } else { builder.setCurrentContentClaim( (String) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_CONTAINER),
final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder() .setEventType(ProvenanceEventType.DOWNLOAD) .setFlowFileUUID(provEvent.getFlowFileUuid()) .setAttributes(provEvent.getAttributes(), Collections.emptyMap()) .setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size) .setTransitUri(requestUri) .setEventTime(System.currentTimeMillis()) .setFlowFileEntryDate(provEvent.getFlowFileEntryDate()) .setLineageStartDate(provEvent.getLineageStartDate()) .setComponentType(flowManager.getRootGroup().getName()) .setComponentId(flowManager.getRootGroupId()) .setDetails("Download of " + (direction == ContentDirection.INPUT ? "Input" : "Output") + " Content requested by " + requestor + " for Provenance Event " + provEvent.getEventId()) .build();
private StandardProvenanceEventRecord readPreVersion6Record(final DataInputStream dis, final int serializationVersion) throws IOException { final long startOffset = getBytesConsumed(); final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventType(eventType); builder.setEventTime(dis.readLong()); builder.setEventDuration(dis.readLong()); builder.setComponentId(readNullableString(dis)); builder.setComponentType(readNullableString(dis)); builder.setFlowFileUUID(readNullableString(dis)); builder.addParentUuid(dis.readUTF()); builder.addChildUuid(dis.readUTF()); builder.setSourceSystemFlowFileIdentifier(readNullableString(dis)); builder.setTransitUri(sourceSystemUri); } else if (destinationSystemUri != null) { builder.setTransitUri(destinationSystemUri); builder.setAlternateIdentifierUri(readNullableString(dis)); builder.setFlowFileEntryDate(System.currentTimeMillis()); builder.setLineageStartDate(-1L); builder.setAttributes(Collections.<String, String>emptyMap(), attrs); builder.setCurrentContentClaim(null, null, null, null, fileSize);
@Override public ProvenanceEventBuilder copy() { final Builder copy = new Builder(); copy.eventTime = eventTime; copy.entryDate = entryDate;
final StandardProvenanceEventRecord.Builder recordBuilder = new StandardProvenanceEventRecord.Builder().fromEvent(rawEvent); if (repoRecord.getCurrent() != null && repoRecord.getCurrentClaim() != null) { final ContentClaim currentClaim = repoRecord.getCurrentClaim(); recordBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), currentOffset + currentClaim.getOffset(), size); recordBuilder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), originalOffset + originalClaim.getOffset(), originalSize); recordBuilder.setSourceQueueIdentifier(originalQueue.getIdentifier()); recordBuilder.setAttributes(repoRecord.getOriginalAttributes(), repoRecord.getUpdatedAttributes()); if (rawEvent.getEventDuration() < 0) { recordBuilder.setEventDuration(TimeUnit.NANOSECONDS.toMillis(commitNanos - repoRecord.getStartNanos())); return recordBuilder.build();