@Override public String toString() { return getValue(); } };
@Override public String getQueryIdentifier() { return query.getIdentifier(); }
@Override public boolean equals(final Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (!(obj instanceof SearchableField)) { return false; } final SearchableField other = (SearchableField) obj; return attribute == other.isAttribute() && this.searchableName.equals(other.getSearchableFieldName()); } }
public static org.apache.lucene.search.Query convertQuery(final org.apache.nifi.provenance.search.Query query) { if (query.getStartDate() == null && query.getEndDate() == null && query.getSearchTerms().isEmpty()) { return new MatchAllDocsQuery(); } final BooleanQuery luceneQuery = new BooleanQuery(); for (final SearchTerm searchTerm : query.getSearchTerms()) { final String searchValue = searchTerm.getValue(); if (searchValue == null) { throw new IllegalArgumentException("Empty search value not allowed (for term '" + searchTerm.getSearchableField().getFriendlyName() + "')"); } if (searchValue.contains("*") || searchValue.contains("?")) { luceneQuery.add(new BooleanClause(new WildcardQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } else { luceneQuery.add(new BooleanClause(new TermQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } } final Long minBytes = query.getMinFileSize() == null ? null : DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); final Long maxBytes = query.getMaxFileSize() == null ? null : DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (minBytes != null || maxBytes != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.FileSize.getSearchableFieldName(), minBytes, maxBytes, true, true), Occur.MUST); } final Long minDateTime = query.getStartDate() == null ? null : query.getStartDate().getTime(); final Long maxDateTime = query.getEndDate() == null ? null : query.getEndDate().getTime(); if (maxDateTime != null || minDateTime != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.EventTime.getSearchableFieldName(), minDateTime, maxDateTime, true, true), Occur.MUST); } return luceneQuery; }
@Override public Optional<List<Long>> evaluate(final Query query) { if (query.getMaxResults() > 1000) { // If query max results > 1000 then we know we don't have enough results. So just return empty. return Optional.empty(); } final List<SearchTerm> terms = query.getSearchTerms(); if (terms.size() != 1) { return Optional.empty(); } final SearchTerm term = terms.get(0); if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) { return Optional.empty(); } if (query.getEndDate() != null || query.getStartDate() != null) { return Optional.empty(); } final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue()); if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) { return Optional.empty(); } List<Long> eventIds = ringBuffer.asList(); if (eventIds.size() > query.getMaxResults()) { eventIds = eventIds.subList(0, query.getMaxResults()); } return Optional.of(eventIds); }
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } } if (result.getError() != null) { throw new IOException(result.getError()); } logger.info("{} got {} hits", query, result.getTotalHitCount()); return result; }
@Override public Optional<List<Long>> evaluate(final Query query) { if (latestRecords.getSize() < query.getMaxResults()) { return Optional.empty(); } if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) { final List<Long> eventList = latestRecords.asList(); if (eventList.size() > query.getMaxResults()) { return Optional.of(eventList.subList(0, query.getMaxResults())); } else { return Optional.of(eventList); } } else { return Optional.empty(); } }
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } if (result.getError() != null) { throw new IOException(result.getError()); } return result; }
final Query query = new Query(provenanceDto.getId()); field = SearchableFields.newSearchableAttribute(searchTerm.getKey()); query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue())); query.setStartDate(requestDto.getStartDate()); query.setEndDate(requestDto.getEndDate()); query.setMinFileSize(requestDto.getMinimumFileSize()); query.setMaxFileSize(requestDto.getMaximumFileSize()); query.setMaxResults(requestDto.getMaxResults()); return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
searchableField.setId(field.getIdentifier()); searchableField.setField(field.getSearchableFieldName()); searchableField.setLabel(field.getFriendlyName()); searchableField.setType(field.getFieldType().name()); searchableFieldNames.add(searchableField); for (final SearchableField searchableAttr : searchableAttributes) { final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO(); searchableAttribute.setId(searchableAttr.getIdentifier()); searchableAttribute.setField(searchableAttr.getSearchableFieldName()); searchableAttribute.setLabel(searchableAttr.getFriendlyName()); searchableAttribute.setType(searchableAttr.getFieldType().name()); searchableFieldNames.add(searchableAttribute);
@Override public boolean isFinished() { readLock.lock(); try { return numCompletedSteps >= numSteps || canceled || matchingRecords.size() >= query.getMaxResults(); } finally { readLock.unlock(); } }
private void validate(final Query query) { final int numQueries = querySubmissionMap.size(); if (numQueries > MAX_UNDELETED_QUERY_RESULTS) { purgeObsoleteQueries(); if (querySubmissionMap.size() > MAX_UNDELETED_QUERY_RESULTS) { throw new IllegalStateException("Cannot process query because there are currently " + numQueries + " queries whose results have not " + "been deleted due to poorly behaving clients not issuing DELETE requests. Please try again later."); } } if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) { throw new IllegalArgumentException("Query End Time cannot be before Query Start Time"); } }
@Override public QuerySubmission retrieveQuerySubmission(final String queryIdentifier, final NiFiUser user) { final QuerySubmission submission = querySubmissionMap.get(queryIdentifier); final String userId = submission.getSubmitterIdentity(); if (user == null && userId == null) { return submission; } if (user == null) { throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because no user id was provided"); } if (userId == null || userId.equals(user.getIdentity())) { return submission; } throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because " + user.getIdentity() + " is not the user who submitted the request"); }
private BooleanQuery buildLineageQuery(final Collection<String> flowFileUuids) { // Create a query for all Events related to the FlowFiles of interest. We do this by adding all ID's as // "SHOULD" clauses and then setting the minimum required to 1. final BooleanQuery lineageQuery; if (flowFileUuids == null || flowFileUuids.isEmpty()) { lineageQuery = null; } else { lineageQuery = new BooleanQuery(); for (final String flowFileUuid : flowFileUuids) { lineageQuery.add(new TermQuery(new Term(SearchableFields.FlowFileUUID.getSearchableFieldName(), flowFileUuid)), Occur.SHOULD); } lineageQuery.setMinimumNumberShouldMatch(1); } return lineageQuery; }
/** * Deletes the query with the specified id. * * @param provenanceId id */ public void deleteProvenanceQuery(final String provenanceId) { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser()); if (querySubmission != null) { querySubmission.cancel(); } }
private void addField(final Document doc, final SearchableField field, final String value) { if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) { return; } doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO)); }
@Override public long getTotalHitCount() { readLock.lock(); try { // Because we filter the results based on the user's permissions, // we don't want to indicate that the total hit count is 1,000+ when we // have 0 matching records, for instance. So, if we have fewer matching // records than the max specified by the query, it is either the case that // we truly don't have enough records to reach the max results, or that // the user is not authorized to see some of the results. Either way, // we want to report the number of events that we find AND that the user // is allowed to see, so we report matching record count, or up to max results. if (matchingRecords.size() < query.getMaxResults()) { return matchingRecords.size(); } else { return query.getMaxResults(); } } finally { readLock.unlock(); } }
@Override public QuerySubmission retrieveQuerySubmission(final String queryIdentifier, final NiFiUser user) { final QuerySubmission submission = querySubmissionMap.get(queryIdentifier); final String userId = submission.getSubmitterIdentity(); if (user == null && userId == null) { return submission; } if (user == null) { throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because no user id was provided in the provenance request."); } if (userId == null || userId.equals(user.getIdentity())) { return submission; } throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because " + user.getIdentity() + " is not the user who submitted the request."); }
private void addField(final Document doc, final SearchableField field, final String value, final Store store) { if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) { return; } doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), store)); }
@Override public QuerySubmission retrieveQuerySubmission(final String queryIdentifier, final NiFiUser user) { final QuerySubmission submission = querySubmissionMap.get(queryIdentifier); final String userId = submission.getSubmitterIdentity(); if (user == null && userId == null) { return submission; } if (user == null) { throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because no user id was provided in the provenance request."); } if (userId == null || userId.equals(user.getIdentity())) { return submission; } throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because " + user.getIdentity() + " is not the user who submitted the request."); }