@Override public boolean select(final ProvenanceEventRecord event) { return event.getEventId() == id; } }, 1);
@Override public long getEventIdentifier() { return record.getEventId(); }
@Override public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) { return Long.compare(o2.getEventId(), o1.getEventId()); } }
@Override public List<ProvenanceEventRecord> getProvenanceEvents(long firstEventId, int maxRecords) throws IOException { if (firstEventId < 0 || maxRecords < 1) { throw new IllegalArgumentException(); } final List<ProvenanceEventRecord> records = new ArrayList<>(); for (final ProvenanceEventRecord record : provenanceRecords) { if (record.getEventId() >= firstEventId) { records.add(record); if (records.size() >= maxRecords) { return records; } } } return records; }
@Override public ProvenanceEventRecord getEvent(final long id) throws IOException { final List<ProvenanceEventRecord> records = getEvents(id, 1); if (records.isEmpty()) { return null; } final ProvenanceEventRecord record = records.get(0); if (record.getEventId() != id) { return null; } return record; }
@Override public boolean select(final ProvenanceEventRecord value) { if (!isAuthorized(value, user)) { return false; } return value.getEventId() >= firstRecordId; } }, maxRecords);
@Override public Long getMaxEventId() { final ProvenanceEventRecord newest = ringBuffer.getNewestElement(); return (newest == null) ? null : newest.getEventId(); }
@Override public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) { // Sort on Event Time, then Event ID. final int eventTimeComparison = Long.compare(o1.getEventTime(), o2.getEventTime()); if (eventTimeComparison == 0) { return Long.compare(o1.getEventId(), o2.getEventId()); } else { return eventTimeComparison; } } });
private long updateLastEventId(final List<ProvenanceEventRecord> events, final StateManager stateManager) { if (events == null || events.isEmpty()) { return firstEventId; } // Store the id of the last event so we know where we left off final ProvenanceEventRecord lastEvent = events.get(events.size() - 1); final String lastEventId = String.valueOf(lastEvent.getEventId()); try { Map<String, String> newMapOfState = new HashMap<>(); newMapOfState.put(LAST_EVENT_ID_KEY, lastEventId); stateManager.setState(newMapOfState, Scope.LOCAL); } catch (final IOException ioe) { logger.error("Failed to update state to {} due to {}; this could result in events being re-sent after a restart. The message of {} was: {}", new Object[]{lastEventId, ioe, ioe, ioe.getMessage()}, ioe); } return lastEvent.getEventId() + 1; }
private List<LineageNode> findParentEvents(Map<String, List<LineageNode>> lineageTree, ProvenanceEventRecord event) { List<LineageNode> parentNodes = lineageTree.get(String.valueOf(event.getEventId())); return parentNodes == null || parentNodes.isEmpty() ? null : parentNodes.stream() // In case it's not a provenance event (i.e. FLOWFILE_NODE), get one level higher parents. .flatMap(n -> !LineageNodeType.PROVENANCE_EVENT_NODE.equals(n.getNodeType()) ? lineageTree.get(n.getIdentifier()).stream() : Stream.of(n)) .collect(Collectors.toList()); }
private static String stringify(final ProvenanceEventRecord event, final int index, final long byteOffset) { final StringBuilder sb = new StringBuilder(); sb.append("Event Index in File = ").append(index).append(", Byte Offset = ").append(byteOffset); sb.append("\n\t").append("Event ID = ").append(event.getEventId()); sb.append("\n\t").append("Event Type = ").append(event.getEventType()); sb.append("\n\t").append("Event Time = ").append(new Date(event.getEventTime())); sb.append("\n\t").append("Event UUID = ").append(event.getFlowFileUuid()); sb.append("\n\t").append("Component ID = ").append(event.getComponentId()); sb.append("\n\t").append("Event ID = ").append(event.getComponentType()); sb.append("\n\t").append("Transit URI = ").append(event.getTransitUri()); sb.append("\n\t").append("Parent IDs = ").append(event.getParentUuids()); sb.append("\n\t").append("Child IDs = ").append(event.getChildUuids()); sb.append("\n\t").append("Previous Attributes = ").append(event.getPreviousAttributes()); sb.append("\n\t").append("Updated Attributes = ").append(event.getUpdatedAttributes()); return sb.toString(); } }
private ProvenanceEventRecord findPreviousProvenanceEvent(AnalysisContext context, ProvenanceEventRecord event) { final ComputeLineageResult lineage = context.queryLineage(event.getEventId()); if (lineage == null) { logger.warn("Lineage was not found: {}", new Object[]{event}); return null; } // If no previous provenance node found due to expired or other reasons, just log a warning msg and do nothing. final LineageNode previousProvenanceNode = traverseLineage(lineage, String.valueOf(event.getEventId())); if (previousProvenanceNode == null) { logger.warn("Traverse lineage could not find any preceding provenance event node: {}", new Object[]{event}); return null; } final long previousEventId = Long.parseLong(previousProvenanceNode.getIdentifier()); return context.getProvenanceEvent(previousEventId); }
@Override public Optional<ProvenanceEventRecord> getEvent(final long id) throws IOException { final Optional<File> option = getPathForEventId(id); if (!option.isPresent()) { return Optional.empty(); } try (final RecordReader reader = recordReaderFactory.newRecordReader(option.get(), Collections.emptyList(), config.getMaxAttributeChars())) { final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(id); if (!eventOption.isPresent()) { return eventOption; } // If an event is returned, the event may be the one we want, or it may be an event with a // higher event ID, if the desired event is not in the record reader. So we need to get the // event and check the Event ID to know whether to return the empty optional or the Optional // that was returned. final ProvenanceEventRecord event = eventOption.get(); if (event.getEventId() == id) { return eventOption; } else { return Optional.empty(); } } }
void performMaintenance() { try { final List<ProvenanceEventRecord> firstEvents = eventStore.getEvents(0, 1); final long earliestEventTime; if (firstEvents.isEmpty()) { earliestEventTime = System.currentTimeMillis(); logger.debug("Found no events in the Provenance Repository. In order to perform maintenace of the indices, " + "will assume that the first event time is now ({})", System.currentTimeMillis()); } else { final ProvenanceEventRecord firstEvent = firstEvents.get(0); earliestEventTime = firstEvent.getEventTime(); logger.debug("First Event Time is {} ({}) with Event ID {}; will delete any Lucene Index that is older than this", earliestEventTime, new Date(earliestEventTime), firstEvent.getEventId()); } final List<File> indicesBeforeEarliestEvent = directoryManager.getDirectoriesBefore(earliestEventTime); for (final File index : indicesBeforeEarliestEvent) { logger.debug("Index directory {} is now expired. Attempting to remove index", index); tryDeleteIndex(index); } } catch (final Exception e) { logger.error("Failed to perform background maintenance procedures", e); eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to perform maintenance of Provenance Repository. See logs for more information."); } }
public PlaceholderProvenanceEvent(final ProvenanceEventRecord original) { this.componentId = original.getComponentId(); this.eventId = original.getEventId(); this.eventTime = original.getEventTime(); this.flowFileUuid = original.getFlowFileUuid(); }
@Override public String toString() { return "Event[ID=" + record.getEventId() + ", Type=" + record.getEventType() + ", UUID=" + record.getFlowFileUuid() + ", Component=" + record.getComponentId() + "]"; } }
logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId()); } else { final File indexDir;
@Override public synchronized StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException { if (isDirty()) { throw new IOException("Cannot update Provenance Repository because this Record Writer has already failed to write to the Repository"); } try { final long recordIdentifier = record.getEventId() == -1L ? idGenerator.getAndIncrement() : record.getEventId(); final long startBytes = byteCountingOut.getBytesWritten(); ensureStreamState(recordIdentifier, startBytes); writeRecord(record, recordIdentifier, out); recordCount++; final long bytesWritten = byteCountingOut.getBytesWritten(); final long serializedLength = bytesWritten - startBytes; final TocWriter tocWriter = getTocWriter(); final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex(); final String storageLocation = getStorageLocation(); return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, bytesWritten); } catch (final IOException ioe) { markDirty(); throw ioe; } }
@Override public long getMaxEventId() throws IOException { if (tocReader != null) { final long lastBlockOffset = tocReader.getLastBlockOffset(); skipToBlock(tocReader.getBlockIndex(lastBlockOffset)); } ProvenanceEventRecord record; ProvenanceEventRecord lastRecord = null; try { while ((record = nextRecord()) != null) { lastRecord = record; } } catch (final EOFException eof) { // This can happen if we stop NIFi while the record is being written. // This is OK, we just ignore this record. The session will not have been // committed, so we can just process the FlowFile again. } return lastRecord == null ? -1L : lastRecord.getEventId(); }
final String platform, final String nodeIdentifier) { addField(builder, "eventId", UUID.randomUUID().toString()); addField(builder, "eventOrdinal", event.getEventId()); addField(builder, "eventType", event.getEventType().name()); addField(builder, "timestampMillis", event.getEventTime()); final String urlPrefix = urlString.substring(0, urlString.length() - DESTINATION_URL_PATH.length()); final String contentUriBase = urlPrefix + "/nifi-api/provenance-events/" + event.getEventId() + "/content/"; final String nodeIdSuffix = nodeIdentifier == null ? "" : "?clusterNodeId=" + nodeIdentifier; addField(builder, "contentURI", contentUriBase + "output" + nodeIdSuffix);