@Override public Optional<List<Long>> evaluate(final Query query) { if (latestRecords.getSize() < query.getMaxResults()) { return Optional.empty(); } if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) { final List<Long> eventList = latestRecords.asList(); if (eventList.size() > query.getMaxResults()) { return Optional.of(eventList.subList(0, query.getMaxResults())); } else { return Optional.of(eventList); } } else { return Optional.empty(); } }
@Override public String getQueryIdentifier() { return query.getIdentifier(); }
private void validate(final Query query) { final int numQueries = querySubmissionMap.size(); if (numQueries > MAX_UNDELETED_QUERY_RESULTS) { purgeObsoleteQueries(); if (querySubmissionMap.size() > MAX_UNDELETED_QUERY_RESULTS) { throw new IllegalStateException("Cannot process query because there are currently " + numQueries + " queries whose results have not " + "been deleted due to poorly behaving clients not issuing DELETE requests. Please try again later."); } } if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) { throw new IllegalArgumentException("Query End Time cannot be before Query Start Time"); } }
requestDto.setStartDate(query.getStartDate()); requestDto.setEndDate(query.getEndDate()); requestDto.setMinimumFileSize(query.getMinFileSize()); requestDto.setMaximumFileSize(query.getMaxFileSize()); requestDto.setMaxResults(query.getMaxResults()); if (query.getSearchTerms() != null) { final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue()); provenanceDto.setId(query.getIdentifier()); provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime()); provenanceDto.setExpiration(queryResult.getExpiration());
public static org.apache.lucene.search.Query convertQuery(final org.apache.nifi.provenance.search.Query query) { if (query.getStartDate() == null && query.getEndDate() == null && query.getSearchTerms().isEmpty()) { return new MatchAllDocsQuery(); } final BooleanQuery luceneQuery = new BooleanQuery(); for (final SearchTerm searchTerm : query.getSearchTerms()) { final String searchValue = searchTerm.getValue(); if (searchValue == null) { throw new IllegalArgumentException("Empty search value not allowed (for term '" + searchTerm.getSearchableField().getFriendlyName() + "')"); } if (searchValue.contains("*") || searchValue.contains("?")) { luceneQuery.add(new BooleanClause(new WildcardQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } else { luceneQuery.add(new BooleanClause(new TermQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } } final Long minBytes = query.getMinFileSize() == null ? null : DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); final Long maxBytes = query.getMaxFileSize() == null ? null : DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (minBytes != null || maxBytes != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.FileSize.getSearchableFieldName(), minBytes, maxBytes, true, true), Occur.MUST); } final Long minDateTime = query.getStartDate() == null ? null : query.getStartDate().getTime(); final Long maxDateTime = query.getEndDate() == null ? null : query.getEndDate().getTime(); if (maxDateTime != null || minDateTime != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.EventTime.getSearchableFieldName(), minDateTime, maxDateTime, true, true), Occur.MUST); } return luceneQuery; }
public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final NiFiUser user, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException { if (retrievedCount.get() >= provenanceQuery.getMaxResults()) { final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1); sqr.update(Collections.<ProvenanceEventRecord> emptyList(), 0L); indexDirectory, provenanceQuery, provenanceQuery.getMaxResults()); if (provenanceQuery.getStartDate() == null || provenanceQuery.getStartDate().getTime() < firstEventTimestamp) { provenanceQuery.setStartDate(new Date(firstEventTimestamp)); if (provenanceQuery.getEndDate() == null) { provenanceQuery.setEndDate(new Date()); final TopDocs topDocs = searcher.getIndexSearcher().search(luceneQuery, provenanceQuery.getMaxResults()); final long finishSearch = System.nanoTime(); final long searchNanos = finishSearch - searchStartNanos; provenanceQuery.getMaxResults(), maxAttributeChars);
@Override public boolean isFinished() { readLock.lock(); try { return numCompletedSteps >= numSteps || canceled || matchingRecords.size() >= query.getMaxResults(); } finally { readLock.unlock(); } }
final Query query = new Query(provenanceDto.getId()); field = SearchableFields.newSearchableAttribute(searchTerm.getKey()); query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue())); query.setStartDate(requestDto.getStartDate()); query.setEndDate(requestDto.getEndDate()); query.setMinFileSize(requestDto.getMinimumFileSize()); query.setMaxFileSize(requestDto.getMaximumFileSize()); query.setMaxResults(requestDto.getMaxResults());
if (query.getStartDate() != null && query.getStartDate().getTime() > event.getEventTime()) { return false; if (query.getEndDate() != null && query.getEndDate().getTime() < event.getEventTime()) { return false; if (query.getMaxFileSize() != null) { final long maxFileSize = DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (event.getFileSize() > maxFileSize) { return false; if (query.getMinFileSize() != null) { final long minFileSize = DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); if (event.getFileSize() < minFileSize) { return false; for (final SearchTerm searchTerm : query.getSearchTerms()) { final SearchableField searchableField = searchTerm.getSearchableField(); final String searchValue = searchTerm.getValue();
@Override public long getTotalHitCount() { readLock.lock(); try { // Because we filter the results based on the user's permissions, // we don't want to indicate that the total hit count is 1,000+ when we // have 0 matching records, for instance. So, if we have fewer matching // records than the max specified by the query, it is either the case that // we truly don't have enough records to reach the max results, or that // the user is not authorized to see some of the results. Either way, // we want to report the number of events that we find AND that the user // is allowed to see, so we report matching record count, or up to max results. if (matchingRecords.size() < query.getMaxResults()) { return matchingRecords.size(); } else { return query.getMaxResults(); } } finally { readLock.unlock(); } }
@Override public QuerySubmission submitQuery(final Query query, final NiFiUser user) { if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) { throw new IllegalArgumentException("Query End Time cannot be before Query Start Time"); } final String userId = user == null ? null : user.getIdentity(); if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) { final AsyncQuerySubmission result = new AsyncQuerySubmission(query, 1, userId); queryExecService.submit(new QueryRunnable(ringBuffer, createFilter(query, user), query.getMaxResults(), result)); querySubmissionMap.put(query.getIdentifier(), result); return result; } final AsyncQuerySubmission result = new AsyncQuerySubmission(query, 1, userId); querySubmissionMap.put(query.getIdentifier(), result); queryExecService.submit(new QueryRunnable(ringBuffer, createFilter(query, user), query.getMaxResults(), result)); return result; }
if (matchingRecords.size() > query.getMaxResults()) { final Iterator<ProvenanceEventRecord> itr = matchingRecords.iterator(); for (int i = 0; i < query.getMaxResults(); i++) { itr.next(); updateExpiration(); if (numCompletedSteps >= numSteps || this.matchingRecords.size() >= query.getMaxResults()) { final long searchNanos = System.nanoTime() - creationNanos; queryTime = TimeUnit.MILLISECONDS.convert(searchNanos, TimeUnit.NANOSECONDS);
@Override public String getQueryIdentifier() { return query.getIdentifier(); }
@Override public Optional<List<Long>> evaluate(final Query query) { if (query.getMaxResults() > 1000) { // If query max results > 1000 then we know we don't have enough results. So just return empty. return Optional.empty(); } final List<SearchTerm> terms = query.getSearchTerms(); if (terms.size() != 1) { return Optional.empty(); } final SearchTerm term = terms.get(0); if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) { return Optional.empty(); } if (query.getEndDate() != null || query.getStartDate() != null) { return Optional.empty(); } final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue()); if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) { return Optional.empty(); } List<Long> eventIds = ringBuffer.asList(); if (eventIds.size() > query.getMaxResults()) { eventIds = eventIds.subList(0, query.getMaxResults()); } return Optional.of(eventIds); }
final int maxResults = query.getMaxResults(); final long startIndex = Math.max(maxEventId - query.getMaxResults(), 0L);
if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) { throw new IllegalArgumentException("Query End Time cannot be before Query Start Time"); if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) { final AsyncQuerySubmission result = new AsyncQuerySubmission(query, 1, userId); if (latestRecords.getSize() >= query.getMaxResults()) { final List<ProvenanceEventRecord> latestList = filterUnauthorizedEvents(latestRecords.asList(), user); final List<ProvenanceEventRecord> trimmed; if (latestList.size() > query.getMaxResults()) { trimmed = latestList.subList(latestList.size() - query.getMaxResults(), latestList.size()); } else { trimmed = latestList; querySubmissionMap.put(query.getIdentifier(), result); return result; query.getStartDate() == null ? null : query.getStartDate().getTime(), query.getEndDate() == null ? null : query.getEndDate().getTime()); final AsyncQuerySubmission result = new AsyncQuerySubmission(query, indexDirectories.size(), userId); querySubmissionMap.put(query.getIdentifier(), result);
@Override public boolean isFinished() { readLock.lock(); try { return numCompletedSteps >= numSteps || canceled || matchingRecords.size() >= query.getMaxResults(); } finally { readLock.unlock(); } }
if (eventIdListOption.isPresent()) { final AsyncQuerySubmission submission = new AsyncQuerySubmission(query, 1, userId); querySubmissionMap.put(query.getIdentifier(), submission); query.getStartDate() == null ? null : query.getStartDate().getTime(), query.getEndDate() == null ? null : query.getEndDate().getTime()); querySubmissionMap.put(query.getIdentifier(), submission); logger.debug("Submitting query {} with identifier {} against index directories {}", luceneQuery, query.getIdentifier(), indexDirectories); queryExecutor.submit(new QueryTask(luceneQuery, submission.getResult(), query.getMaxResults(), indexManager, indexDir, eventStore, authorizer, EventTransformer.EMPTY_TRANSFORMER));
@Override public long getTotalHitCount() { readLock.lock(); try { // Because we filter the results based on the user's permissions, // we don't want to indicate that the total hit count is 1,000+ when we // have 0 matching records, for instance. So, if we have fewer matching // records than the max specified by the query, it is either the case that // we truly don't have enough records to reach the max results, or that // the user is not authorized to see some of the results. Either way, // we want to report the number of events that we find AND that the user // is allowed to see, so we report matching record count, or up to max results. if (matchingRecords.size() < query.getMaxResults()) { return matchingRecords.size(); } else { return query.getMaxResults(); } } finally { readLock.unlock(); } }
if (matchingRecords.size() > query.getMaxResults()) { final Iterator<ProvenanceEventRecord> itr = matchingRecords.iterator(); for (int i = 0; i < query.getMaxResults(); i++) { itr.next(); updateExpiration(); if (numCompletedSteps >= numSteps || this.matchingRecords.size() >= query.getMaxResults()) { final long searchNanos = System.nanoTime() - creationNanos; queryTime = TimeUnit.MILLISECONDS.convert(searchNanos, TimeUnit.NANOSECONDS);