@Override public void run() { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (result.isFinished() && result.getExpiration().before(now)) { querySubmissionMap.remove(entry.getKey()); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (result.isFinished() && result.getExpiration().before(now)) { querySubmissionMap.remove(entry.getKey()); } } } }
@Override public void run() { try { final IndexSearch search = new IndexSearch(PersistentProvenanceRepository.this, indexDir, getIndexManager(), maxAttributeChars); final StandardQueryResult queryResult = search.search(query, user, retrievalCount, firstEventTimestamp); submission.getResult().update(queryResult.getMatchingEvents(), queryResult.getTotalHitCount()); } catch (final Throwable t) { logger.error("Failed to query Provenance Repository Index {} due to {}", indexDir, t.toString()); if (logger.isDebugEnabled()) { logger.error("", t); } if (t.getMessage() == null) { submission.getResult().setError(t.toString()); } else { submission.getResult().setError(t.getMessage()); } } } }
/** * Constructs an AsyncQuerySubmission with the given query and the given * number of steps, indicating how many results must be added to this * AsyncQuerySubmission before it is considered finished * * @param query the query to execute * @param numSteps how many steps to include */ public AsyncQuerySubmission(final Query query, final int numSteps, final String submitterId) { this.query = query; this.submitterId = submitterId; queryResult = new StandardQueryResult(query, numSteps); }
if (isFinished()) { return; updateExpiration();
try { events = eventStore.getEvents(eventIds, authorizer, EventTransformer.EMPTY_TRANSFORMER); submission.getResult().update(events, eventIds.size()); } catch (final Exception e) { submission.getResult().setError("Failed to retrieve Provenance Events from store; see logs for more details"); logger.error("Failed to retrieve Provenance Events from store", e); submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); submission.getResult().update(Collections.emptyList(), 0L); } else { Collections.sort(indexDirectories, DirectoryUtils.NEWEST_INDEX_FIRST); submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS); } catch (final InterruptedException e) { Thread.currentThread().interrupt();
submission.getResult().update(Collections.<ProvenanceEventRecord>emptyList(), 0); return; submission.getResult().update(mostRecent, totalNumDocs); } catch (final IOException ioe) { logger.error("Failed to retrieve records from Provenance Repository: " + ioe.toString()); submission.getResult().setError("Failed to retrieve records from Provenance Repository: " + ioe.toString()); } else { submission.getResult().setError("Failed to retrieve records from Provenance Repository: " + ioe.getMessage());
public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final NiFiUser user, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException { if (retrievedCount.get() >= provenanceQuery.getMaxResults()) { final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1); sqr.update(Collections.<ProvenanceEventRecord> emptyList(), 0L); final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1); final Set<ProvenanceEventRecord> matchingRecords; sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0); return sqr; logger.debug("Reading {} records took {} millis for {}", matchingRecords.size(), TimeUnit.NANOSECONDS.toMillis(readRecordsNanos), this); sqr.update(matchingRecords, topDocs.totalHits); sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0); return sqr; } finally {
@Override public boolean awaitCompletion(final long time, final TimeUnit unit) throws InterruptedException { final long finishTime = System.currentTimeMillis() + unit.toMillis(time); synchronized (completionMonitor) { while (!isFinished()) { final long millisToWait = finishTime - System.currentTimeMillis(); if (millisToWait > 0) { completionMonitor.wait(millisToWait); } else { return isFinished(); } } } return isFinished(); }
@Override public void run() { // Retrieve the most recent results and count the total number of matches final AtomicInteger matchingCount = new AtomicInteger(0); final List<ProvenanceEventRecord> matchingRecords = new ArrayList<>(maxRecords); ringBuffer.forEach(new ForEachEvaluator<ProvenanceEventRecord>() { @Override public boolean evaluate(final ProvenanceEventRecord record) { if (filter.select(record)) { if (matchingCount.incrementAndGet() <= maxRecords) { matchingRecords.add(record); } } return true; } }, IterationDirection.BACKWARD); submission.getResult().update(matchingRecords, matchingCount.get()); } }
@Override public void cancel() { this.canceled = true; for (Future<?> queryExecution : this.queryExecutions) { queryExecution.cancel(true); } queryResult.cancel(); }
submission.getResult().update(Collections.<ProvenanceEventRecord>emptyList(), 0); return; submission.getResult().update(mostRecent, totalNumDocs); } catch (final IOException ioe) { logger.error("Failed to retrieve records from Provenance Repository: " + ioe.toString()); submission.getResult().setError("Failed to retrieve records from Provenance Repository: " + ioe.toString()); } else { submission.getResult().setError("Failed to retrieve records from Provenance Repository: " + ioe.getMessage());
if (isFinished()) { return; updateExpiration();
@Override public boolean awaitCompletion(final long time, final TimeUnit unit) throws InterruptedException { final long finishTime = System.currentTimeMillis() + unit.toMillis(time); synchronized (completionMonitor) { while (!isFinished()) { final long millisToWait = finishTime - System.currentTimeMillis(); if (millisToWait > 0) { completionMonitor.wait(millisToWait); } else { return isFinished(); } } } return isFinished(); }
result.getResult().update(Collections.<ProvenanceEventRecord>emptyList(), 0L); maxEventId = 0L; result.getResult().update(trimmed, totalNumDocs); } else { queryExecService.submit(new GetMostRecentRunnable(query, result, user)); result.getResult().update(Collections.<ProvenanceEventRecord>emptyList(), 0L); } else { for (final File indexDir : indexDirectories) {
@Override public void cancel() { this.canceled = true; for (Future<?> queryExecution : this.queryExecutions) { queryExecution.cancel(true); } queryResult.cancel(); }
private void purgeObsoleteQueries() { try { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { queryIterator.remove(); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { lineageIterator.remove(); } } } catch (final Exception e) { logger.error("Failed to expire Provenance Query Results due to {}", e.toString()); logger.error("", e); } } }
/** * Constructs an AsyncQuerySubmission with the given query and the given * number of steps, indicating how many results must be added to this * AsyncQuerySubmission before it is considered finished * * @param query the query to execute * @param numSteps how many steps to include */ public AsyncQuerySubmission(final Query query, final int numSteps, final String submitterId) { this.query = query; this.submitterId = submitterId; queryResult = new StandardQueryResult(query, numSteps); }
@Override public void run() { try { final Date now = new Date(); final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator(); while (queryIterator.hasNext()) { final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next(); final StandardQueryResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { queryIterator.remove(); } } final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator(); while (lineageIterator.hasNext()) { final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next(); final StandardLineageResult result = entry.getValue().getResult(); if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) { lineageIterator.remove(); } } } catch (final Throwable t) { logger.error("Failed to expire Provenance Query Results due to {}", t.toString()); logger.error("", t); } } }