public void index(final StandardProvenanceEventRecord record, final IndexWriter indexWriter, final Integer blockIndex) throws IOException { final Document doc = new Document(); addField(doc, SearchableFields.FlowFileUUID, record.getFlowFileUuid(), Store.NO); addField(doc, SearchableFields.Filename, record.getAttribute(CoreAttributes.FILENAME.key()), Store.NO); addField(doc, SearchableFields.ComponentID, record.getComponentId(), Store.NO); addField(doc, SearchableFields.AlternateIdentifierURI, record.getAlternateIdentifierUri(), Store.NO); addField(doc, SearchableFields.EventType, record.getEventType().name(), Store.NO); addField(doc, SearchableFields.Relationship, record.getRelationship(), Store.NO); addField(doc, SearchableFields.Details, record.getDetails(), Store.NO); addField(doc, SearchableFields.ContentClaimSection, record.getContentClaimSection(), Store.NO); addField(doc, SearchableFields.ContentClaimContainer, record.getContentClaimContainer(), Store.NO); addField(doc, SearchableFields.ContentClaimIdentifier, record.getContentClaimIdentifier(), Store.NO); addField(doc, SearchableFields.SourceQueueIdentifier, record.getSourceQueueIdentifier(), Store.NO); addField(doc, SearchableFields.TransitURI, record.getTransitUri(), Store.NO); addField(doc, searchableField, LuceneUtil.truncateIndexField(record.getAttribute(searchableField.getSearchableFieldName())), Store.NO); final String storageFilename = LuceneUtil.substringBefore(record.getStorageFilename(), "."); doc.add(new LongField(SearchableFields.LineageStartDate.getSearchableFieldName(), record.getLineageStartDate(), Store.NO)); doc.add(new LongField(SearchableFields.EventTime.getSearchableFieldName(), record.getEventTime(), Store.NO)); doc.add(new LongField(SearchableFields.FileSize.getSearchableFieldName(), record.getFileSize(), Store.NO)); doc.add(new StringField(FieldNames.STORAGE_FILENAME, storageFilename, Store.YES)); doc.add(new LongField(FieldNames.STORAGE_FILE_OFFSET, record.getStorageByteOffset(), Store.YES)); } else { doc.add(new IntField(FieldNames.BLOCK_INDEX, blockIndex, Store.YES)); doc.add(new LongField(SearchableFields.Identifier.getSearchableFieldName(), record.getEventId(), Store.YES)); final ProvenanceEventType eventType = record.getEventType();
private StandardProvenanceEventRecord truncateAttributes(final StandardProvenanceEventRecord original) { boolean requireTruncation = false; for (final String updatedAttr : original.getUpdatedAttributes().values()) { if (updatedAttr != null && updatedAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } if (!requireTruncation) { for (final String previousAttr : original.getPreviousAttributes().values()) { if (previousAttr != null && previousAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } } if (!requireTruncation) { return original; } final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder().fromEvent(original); builder.setAttributes(truncateAttributes(original.getPreviousAttributes()), truncateAttributes(original.getUpdatedAttributes())); final StandardProvenanceEventRecord truncated = builder.build(); truncated.setEventId(original.getEventId()); return truncated; }
/** * Returns a unique identifier for the record. By default, it uses * {@link ProvenanceEventRecord#getEventId()} but if it has not been persisted to the * repository, this is {@code -1}, so it constructs a String of the format * {@code <event type>_on_<flowfile UUID>_by_<component UUID>_at_<event time>}. * * @return a String identifying the record for later analysis */ @Override public String getBestEventIdentifier() { if (getEventId() != -1) { return Long.toString(getEventId()); } else { return getEventType().name() + "_on_" + getFlowFileUuid() + "_by_" + getComponentId() + "_at_" + getEventTime(); } }
if (eventId > 0L && other.getEventId() > 0L && eventId != other.getEventId()) { return false; if (different(parentUuids, other.parentUuids)) { return false; if (different(childrenUuids, other.childrenUuids)) { return false; if (different(transitUri, other.transitUri)) { return false; if (different(relationship, other.relationship)) { return false; return !(eventType == ProvenanceEventType.REPLAY && eventTime != other.getEventTime());
@Override public int hashCode() { final int eventTypeCode; if (eventType == ProvenanceEventType.CLONE || eventType == ProvenanceEventType.JOIN || eventType == ProvenanceEventType.FORK) { eventTypeCode = 1472; } else if (eventType == ProvenanceEventType.REPLAY) { eventTypeCode = 21479 + (int) (0x7FFFFFFF & eventTime); // use lower bits of event time. } else { eventTypeCode = 4812 + eventType.hashCode() + 4 * uuid.hashCode(); } return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode()) + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode(); }
@Override public int compare(final StandardProvenanceEventRecord o1, final StandardProvenanceEventRecord o2) { return Long.compare(o1.getEventId(), o2.getEventId()); } });
if (record.getEventTime() < earliestTimestamp) { earliestTimestamp = record.getEventTime(); if (record.getEventId() < minEventId) { minEventId = record.getEventId(); maxId = record.getEventId();
@Override public void registerEvent(final ProvenanceEventRecord event) { final StandardProvenanceEventRecord newRecord; if (event instanceof StandardProvenanceEventRecord) { newRecord = (StandardProvenanceEventRecord) event; } else { newRecord = new StandardProvenanceEventRecord.Builder().fromEvent(event).build(); } newRecord.setEventId(idGenerator.getAndIncrement()); records.add(newRecord); }
private Long getFirstEntryTime(final File provenanceLogFile) { if (provenanceLogFile == null) { return null; } try (final RecordReader reader = RecordReaders.newRecordReader(provenanceLogFile, null, Integer.MAX_VALUE)) { final StandardProvenanceEventRecord firstRecord = reader.nextRecord(); if (firstRecord == null) { return provenanceLogFile.lastModified(); } return firstRecord.getEventTime(); } catch (final FileNotFoundException | EOFException fnf) { return null; // file no longer exists or there's no record in this file } catch (final IOException ioe) { logger.warn("Failed to read first entry in file {} due to {}", provenanceLogFile, ioe.toString()); logger.warn("", ioe); return null; } }
/** * Returns a unique identifier for the record. By default, it uses * {@link ProvenanceEventRecord#getEventId()} but if it has not been persisted to the * repository, this is {@code -1}, so it constructs a String of the format * {@code <event type>_on_<flowfile UUID>_by_<component UUID>_at_<event time>}. * * @return a String identifying the record for later analysis */ @Override public String getBestEventIdentifier() { if (getEventId() != -1) { return Long.toString(getEventId()); } else { return getEventType().name() + "_on_" + getFlowFileUuid() + "_by_" + getComponentId() + "_at_" + getEventTime(); } }
if (eventId > 0L && other.getEventId() > 0L && eventId != other.getEventId()) { return false; if (different(parentUuids, other.parentUuids)) { return false; if (different(childrenUuids, other.childrenUuids)) { return false; if (different(transitUri, other.transitUri)) { return false; if (different(relationship, other.relationship)) { return false; return !(eventType == ProvenanceEventType.REPLAY && eventTime != other.getEventTime());
protected Optional<StandardProvenanceEventRecord> readToEvent(final long eventId, final DataInputStream dis, final int serializationVerison) throws IOException { StandardProvenanceEventRecord event; while ((event = nextRecord()) != null) { if (event.getEventId() >= eventId) { return Optional.of(event); } } return Optional.empty(); }
if (record.getEventTime() < earliestTimestamp) { earliestTimestamp = record.getEventTime(); if (record.getEventId() < minEventId) { minEventId = record.getEventId(); maxId = record.getEventId();
private StandardProvenanceEventRecord readRecord(final DataInputStream in, final long eventId, final long startOffset, final int recordLength) throws IOException { final InputStream limitedIn = new LimitingInputStream(in, recordLength); final Record eventRecord = recordReader.readRecord(limitedIn); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes); deserializedEvent.setEventId(eventId); return deserializedEvent; }
private long determineFirstEventTimestamp() { // Get the timestamp of the first event in the first Provenance Event Log File and the ID of the last event // in the event file. final List<File> logFiles = getSortedLogFiles(); if (logFiles.isEmpty()) { return 0L; } for (final File logFile : logFiles) { try (final RecordReader reader = RecordReaders.newRecordReader(logFile, null, Integer.MAX_VALUE)) { final StandardProvenanceEventRecord event = reader.nextRecord(); if (event != null) { return event.getEventTime(); } } catch (final IOException ioe) { logger.warn("Failed to obtain timestamp of first event from Provenance Event Log File {}", logFile); } } return 0L; }
@Override public int hashCode() { final int eventTypeCode; if (eventType == ProvenanceEventType.CLONE || eventType == ProvenanceEventType.JOIN || eventType == ProvenanceEventType.FORK) { eventTypeCode = 1472; } else if (eventType == ProvenanceEventType.REPLAY) { eventTypeCode = 21479 + (int) (0x7FFFFFFF & eventTime); // use lower bits of event time. } else { eventTypeCode = 4812 + eventType.hashCode() + 4 * uuid.hashCode(); } return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode()) + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode(); }
private StandardProvenanceEventRecord truncateAttributes(final StandardProvenanceEventRecord original) { boolean requireTruncation = false; for (final String updatedAttr : original.getUpdatedAttributes().values()) { if (updatedAttr != null && updatedAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } if (!requireTruncation) { for (final String previousAttr : original.getPreviousAttributes().values()) { if (previousAttr != null && previousAttr.length() > maxAttributeChars) { requireTruncation = true; break; } } } if (!requireTruncation) { return original; } final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder().fromEvent(original); builder.setAttributes(truncateAttributes(original.getPreviousAttributes()), truncateAttributes(original.getUpdatedAttributes())); final StandardProvenanceEventRecord truncated = builder.build(); truncated.setEventId(original.getEventId()); return truncated; }
if (pushbackEvent != null) { final StandardProvenanceEventRecord previousPushBack = pushbackEvent; if (previousPushBack.getEventId() >= eventId) { return Optional.of(previousPushBack); } else {
private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException { try { final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength); byte[] encryptedSerializedBytes = new byte[recordLength]; DataInputStream encryptedInputStream = new DataInputStream(limitedIn); encryptedInputStream.readFully(encryptedSerializedBytes); byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId)); InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes); final Record eventRecord = getRecordReader().readRecord(plainStream); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes()); deserializedEvent.setEventId(eventId); return deserializedEvent; } catch (EncryptionException e) { logger.error("Encountered an error reading the record: ", e); throw new IOException(e); } }
try (final RecordReader reader = RecordReaders.newRecordReader(firstLogFile, null, Integer.MAX_VALUE)) { final StandardProvenanceEventRecord event = reader.nextRecord(); earliestEventTime = event.getEventTime(); maxEventId = reader.getMaxEventId(); } catch (final IOException ioe) {