@Override public boolean equals(final Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (!(obj instanceof SearchableField)) { return false; } final SearchableField other = (SearchableField) obj; return attribute == other.isAttribute() && this.searchableName.equals(other.getSearchableFieldName()); } }
searchableField.setId(field.getIdentifier()); searchableField.setField(field.getSearchableFieldName()); searchableField.setLabel(field.getFriendlyName()); searchableField.setType(field.getFieldType().name()); searchableFieldNames.add(searchableField); for (final SearchableField searchableAttr : searchableAttributes) { final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO(); searchableAttribute.setId(searchableAttr.getIdentifier()); searchableAttribute.setField(searchableAttr.getSearchableFieldName()); searchableAttribute.setLabel(searchableAttr.getFriendlyName()); searchableAttribute.setType(searchableAttr.getFieldType().name()); searchableFieldNames.add(searchableAttribute);
private BooleanQuery buildLineageQuery(final Collection<String> flowFileUuids) { // Create a query for all Events related to the FlowFiles of interest. We do this by adding all ID's as // "SHOULD" clauses and then setting the minimum required to 1. final BooleanQuery lineageQuery; if (flowFileUuids == null || flowFileUuids.isEmpty()) { lineageQuery = null; } else { lineageQuery = new BooleanQuery(); for (final String flowFileUuid : flowFileUuids) { lineageQuery.add(new TermQuery(new Term(SearchableFields.FlowFileUUID.getSearchableFieldName(), flowFileUuid)), Occur.SHOULD); } lineageQuery.setMinimumNumberShouldMatch(1); } return lineageQuery; }
public static org.apache.lucene.search.Query convertQuery(final org.apache.nifi.provenance.search.Query query) { if (query.getStartDate() == null && query.getEndDate() == null && query.getSearchTerms().isEmpty()) { return new MatchAllDocsQuery(); } final BooleanQuery luceneQuery = new BooleanQuery(); for (final SearchTerm searchTerm : query.getSearchTerms()) { final String searchValue = searchTerm.getValue(); if (searchValue == null) { throw new IllegalArgumentException("Empty search value not allowed (for term '" + searchTerm.getSearchableField().getFriendlyName() + "')"); } if (searchValue.contains("*") || searchValue.contains("?")) { luceneQuery.add(new BooleanClause(new WildcardQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } else { luceneQuery.add(new BooleanClause(new TermQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } } final Long minBytes = query.getMinFileSize() == null ? null : DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); final Long maxBytes = query.getMaxFileSize() == null ? null : DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (minBytes != null || maxBytes != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.FileSize.getSearchableFieldName(), minBytes, maxBytes, true, true), Occur.MUST); } final Long minDateTime = query.getStartDate() == null ? null : query.getStartDate().getTime(); final Long maxDateTime = query.getEndDate() == null ? null : query.getEndDate().getTime(); if (maxDateTime != null || minDateTime != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.EventTime.getSearchableFieldName(), minDateTime, maxDateTime, true, true), Occur.MUST); } return luceneQuery; }
final String searchValue = searchTerm.getValue(); if (searchableField.isAttribute()) { final String attributeName = searchableField.getIdentifier();
final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
.map(doc -> doc.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue()) .collect(Collectors.toList());
private void addField(final Document doc, final SearchableField field, final String value) { if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) { return; } doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO)); }
flowFileIdQuery = new BooleanQuery(); for (final String flowFileUuid : flowFileUuids) { flowFileIdQuery.add(new TermQuery(new Term(SearchableFields.FlowFileUUID.getSearchableFieldName(), flowFileUuid)), Occur.SHOULD);
private void addField(final Document doc, final SearchableField field, final String value, final Store store) { if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) { return; } doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), store)); }
@Override public int compare(final Document o1, final Document o2) { final String filename1 = o1.get(FieldNames.STORAGE_FILENAME); final String filename2 = o2.get(FieldNames.STORAGE_FILENAME); final int filenameComp = filename1.compareTo(filename2); if (filenameComp != 0) { return filenameComp; } final IndexableField fileOffset1 = o1.getField(FieldNames.BLOCK_INDEX); final IndexableField fileOffset2 = o1.getField(FieldNames.BLOCK_INDEX); if ( fileOffset1 != null && fileOffset2 != null ) { final int blockIndexResult = Long.compare(fileOffset1.numericValue().longValue(), fileOffset2.numericValue().longValue()); if ( blockIndexResult != 0 ) { return blockIndexResult; } final long eventId1 = o1.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); final long eventId2 = o2.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); return Long.compare(eventId1, eventId2); } final long offset1 = o1.getField(FieldNames.STORAGE_FILE_OFFSET).numericValue().longValue(); final long offset2 = o2.getField(FieldNames.STORAGE_FILE_OFFSET).numericValue().longValue(); return Long.compare(offset1, offset2); } });
@Override public boolean equals(final Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (!(obj instanceof SearchableField)) { return false; } final SearchableField other = (SearchableField) obj; return attribute == other.isAttribute() && this.searchableName.equals(other.getSearchableFieldName()); } }
@Override public Optional<List<Long>> evaluate(final Query query) { if (query.getMaxResults() > 1000) { // If query max results > 1000 then we know we don't have enough results. So just return empty. return Optional.empty(); } final List<SearchTerm> terms = query.getSearchTerms(); if (terms.size() != 1) { return Optional.empty(); } final SearchTerm term = terms.get(0); if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) { return Optional.empty(); } if (query.getEndDate() != null || query.getStartDate() != null) { return Optional.empty(); } final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue()); if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) { return Optional.empty(); } List<Long> eventIds = ringBuffer.asList(); if (eventIds.size() > query.getMaxResults()) { eventIds = eventIds.subList(0, query.getMaxResults()); } return Optional.of(eventIds); }
final long eventId = doc.getDocument().getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); if (eventId < minId) { minId = eventId; SearchableFields.Identifier.getSearchableFieldName(), minId, maxId, true, true); indexWriter.getIndexWriter().deleteDocuments(query);
.mapToLong(doc -> doc.getDocument().getField(SearchableFields.EventTime.getSearchableFieldName()).numericValue().longValue()) .min() .getAsLong(); .mapToLong(doc -> doc.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue()) .max() .orElse(-1L);
long getMaxEventId(final String partitionName) { final List<File> allDirectories = getDirectoryManager().getDirectories(0L, Long.MAX_VALUE, partitionName); if (allDirectories.isEmpty()) { return -1L; } Collections.sort(allDirectories, DirectoryUtils.NEWEST_INDEX_FIRST); for (final File directory : allDirectories) { final EventIndexSearcher searcher; try { searcher = indexManager.borrowIndexSearcher(directory); } catch (final IOException ioe) { logger.warn("Unable to read from Index Directory {}. Will assume that the index is incomplete and not consider this index when determining max event ID", directory); continue; } try { final IndexReader reader = searcher.getIndexSearcher().getIndexReader(); final int maxDocId = reader.maxDoc() - 1; final Document document = reader.document(maxDocId); final long eventId = document.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); logger.info("Determined that Max Event ID indexed for Partition {} is approximately {} based on index {}", partitionName, eventId, directory); return eventId; } catch (final IOException ioe) { logger.warn("Unable to search Index Directory {}. Will assume that the index is incomplete and not consider this index when determining max event ID", directory, ioe); } finally { indexManager.returnIndexSearcher(searcher); } } return -1L; }
private ProvenanceEventRecord getRecord(final Document d, final RecordReader reader) throws IOException { final IndexableField blockField = d.getField(FieldNames.BLOCK_INDEX); if ( blockField == null ) { reader.skipTo(getByteOffset(d, reader)); } else { reader.skipToBlock(blockField.numericValue().intValue()); } StandardProvenanceEventRecord record; while ( (record = reader.nextRecord()) != null) { final IndexableField idField = d.getField(SearchableFields.Identifier.getSearchableFieldName()); if ( idField == null || idField.numericValue().longValue() == record.getEventId() ) { break; } } if (record == null) { logger.warn("Failed to read Provenance Event for '" + d + "'. The event file may be missing or corrupted"); } return record; }
addField(doc, searchableField, LuceneUtil.truncateIndexField(record.getAttribute(searchableField.getSearchableFieldName())), Store.NO); doc.add(new LongField(SearchableFields.LineageStartDate.getSearchableFieldName(), record.getLineageStartDate(), Store.NO)); doc.add(new LongField(SearchableFields.EventTime.getSearchableFieldName(), record.getEventTime(), Store.NO)); doc.add(new LongField(SearchableFields.FileSize.getSearchableFieldName(), record.getFileSize(), Store.NO)); doc.add(new StringField(FieldNames.STORAGE_FILENAME, storageFilename, Store.YES)); } else { doc.add(new IntField(FieldNames.BLOCK_INDEX, blockIndex, Store.YES)); doc.add(new LongField(SearchableFields.Identifier.getSearchableFieldName(), record.getEventId(), Store.YES));
addField(doc, searchableField, LuceneUtil.truncateIndexField(record.getAttribute(searchableField.getSearchableFieldName()))); doc.add(new LongField(SearchableFields.LineageStartDate.getSearchableFieldName(), record.getLineageStartDate(), Store.NO)); doc.add(new LongField(SearchableFields.EventTime.getSearchableFieldName(), record.getEventTime(), Store.NO)); doc.add(new LongField(SearchableFields.FileSize.getSearchableFieldName(), record.getFileSize(), Store.NO)); doc.add(new UnIndexedLongField(SearchableFields.Identifier.getSearchableFieldName(), persistedEvent.getEventId()));