@Override public void run() { final List<ProvenanceEventRecord> records = ringBuffer.getSelectedElements(filter); submission.getResult().update(records, records.size()); } }
@Override public void run() { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (result.isFinished() && result.getExpiration().before(now)) { querySubmissionMap.remove(entry.getKey()); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (result.isFinished() && result.getExpiration().before(now)) { querySubmissionMap.remove(entry.getKey()); } } } }
private void purgeObsoleteQueries() { try { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { queryIterator.remove(); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { lineageIterator.remove(); } } } catch (final Exception e) { logger.error("Failed to expire Provenance Query Results due to {}", e.toString()); logger.error("", e); } } }
@Override public void run() { try { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { queryIterator.remove(); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { lineageIterator.remove(); } } } catch (final Throwable t) { logger.error("Failed to expire Provenance Query Results due to {}", t.toString()); logger.error("", t); } } }
private ComputeLineageSubmission submitLineageComputation(final Collection<String> flowFileUuids, final NiFiUser user, final EventAuthorizer eventAuthorizer, final LineageComputationType computationType, final Long eventId, final long startTimestamp, final long endTimestamp) { final List<File> indexDirs = directoryManager.getDirectories(startTimestamp, endTimestamp); final AsyncLineageSubmission submission = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, indexDirs.size(), user == null ? null : user.getIdentity()); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); final BooleanQuery lineageQuery = buildLineageQuery(flowFileUuids); final List<File> indexDirectories = directoryManager.getDirectories(startTimestamp, endTimestamp); if (indexDirectories.isEmpty()) { submission.getResult().update(Collections.emptyList(), 0L); } else { Collections.sort(indexDirectories, DirectoryUtils.OLDEST_INDEX_FIRST); for (final File indexDir : indexDirectories) { queryExecutor.submit(new QueryTask(lineageQuery, submission.getResult(), MAX_LINEAGE_NODES, indexManager, indexDir, eventStore, eventAuthorizer, EventTransformer.PLACEHOLDER_TRANSFORMER)); } } // Some computations will complete very quickly. In this case, we don't want to wait // for the client to submit a second query to obtain the result. Instead, we want to just // wait some short period of time for the computation to complete before returning the submission. try { submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } return submission; }
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user) { final ProvenanceEventRecord event = getEvent(eventId); if (event == null) { final String userId = user == null ? null : user.getIdentity(); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId); result.getResult().setError("Could not find event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, LineageComputationType.FLOWFILE_LINEAGE, eventId); }
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user, final EventAuthorizer eventAuthorizer) { final Optional<ProvenanceEventRecord> eventOption; try { eventOption = eventStore.getEvent(eventId); } catch (final Exception e) { logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information."); return result; } if (!eventOption.isPresent()) { final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Could not find Provenance Event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } final ProvenanceEventRecord event = eventOption.get(); return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, eventAuthorizer, LineageComputationType.FLOWFILE_LINEAGE, eventId, event.getLineageStartDate(), Long.MAX_VALUE); }
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; submission.getResult().setError(ioe.toString()); } else { submission.getResult().setError(ioe.getMessage());
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user) { final ProvenanceEventRecord event; try { event = getEvent(eventId); } catch (final Exception e) { logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information."); return result; } if (event == null) { final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Could not find Provenance Event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, LineageComputationType.FLOWFILE_LINEAGE, eventId, event.getLineageStartDate(), Long.MAX_VALUE); }
private Lineage computeLineage(final Collection<String> flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) throws IOException { final AsyncLineageSubmission submission = submitLineageComputation(flowFileUuids, user, computationType, eventId); final StandardLineageResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } if (result.getError() != null) { throw new IOException(result.getError()); } return new FlowFileLineage(result.getNodes(), result.getEdges()); }
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; submission.getResult().setError(ioe.toString()); } else { submission.getResult().setError(ioe.getMessage());
private Lineage computeLineage(final Collection<String> flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId, final Long startTimestamp, final Long endTimestamp) throws IOException { final AsyncLineageSubmission submission = submitLineageComputation(flowFileUuids, user, computationType, eventId, startTimestamp, endTimestamp); final StandardLineageResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } if (result.getError() != null) { throw new IOException(result.getError()); } return new FlowFileLineage(result.getNodes(), result.getEdges()); }
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e); return submission;
@Override public ComputeLineageSubmission submitExpandParents(final long eventId, final NiFiUser user) { final String userId = user == null ? null : user.getIdentity(); final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; } switch (event.getEventType()) { case JOIN: case FORK: case REPLAY: case CLONE: return submitLineageComputation(event.getParentUuids(), user, LineageComputationType.EXPAND_PARENTS, eventId); default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; } } }
@Override public ComputeLineageSubmission submitExpandChildren(final long eventId, final NiFiUser user) { final String userId = user == null ? null : user.getIdentity(); final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; } switch (event.getEventType()) { case JOIN: case FORK: case REPLAY: case CLONE: return submitLineageComputation(event.getChildUuids(), user, LineageComputationType.EXPAND_CHILDREN, eventId); default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; } } }
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Failed to expand parents for lineage of event with ID " + eventId + " due to: " + e); return submission;