private void deleteBulkRequest(String objectId, String index, String type, String routing, String parent) { if (logger.isTraceEnabled()) { logger.trace("bulkDeleteRequest - objectId: {} - index: {} - type: {} - routing: {} - parent: {}", objectId, index, type, routing, parent); } if (definition.getParentTypes() != null && definition.getParentTypes().contains(type)) { QueryBuilder builder = QueryBuilders.hasParentQuery(type, QueryBuilders.termQuery(MongoDBRiver.MONGODB_ID_FIELD, objectId)); SearchResponse response = esClient.prepareSearch(index).setQuery(builder).setRouting(routing) .addField(MongoDBRiver.MONGODB_ID_FIELD).execute().actionGet(); for (SearchHit hit : response.getHits().getHits()) { getBulkProcessor(index, hit.getType()).deleteBulkRequest(hit.getId(), routing, objectId); } } getBulkProcessor(index, type).deleteBulkRequest(objectId, routing, parent); }
/** * Adds the fields to builder * * @param fieldsToSelect * the fields to select * @param clazz * the clazz * @param metaModel * the meta model * @param builder * the builder */ private void addFieldsToBuilder(String[] fieldsToSelect, Class clazz, MetamodelImpl metaModel, SearchRequestBuilder builder) { if (fieldsToSelect != null && fieldsToSelect.length > 1 && !(fieldsToSelect[1] == null)) { logger.debug("Fields added in query are: "); for (int i = 1; i < fieldsToSelect.length; i++) { logger.debug(i + " : " + fieldsToSelect[i]); builder = builder.addField(((AbstractAttribute) metaModel.entity(clazz).getAttribute(fieldsToSelect[i])) .getJPAColumnName()); } } }
@Override public <E> List<E> getColumnsById(String schemaName, String tableName, String pKeyColumnName, String columnName, Object pKeyColumnValue, Class columnJavaType) { // fetch list ADDRESS_ID for given PERSON_ID QueryBuilder filterBuilder = new TermQueryBuilder(pKeyColumnName, pKeyColumnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()).setTypes(tableName) .setPostFilter(filterBuilder).addField(columnName).execute().actionGet(); SearchHits hits = response.getHits(); List columns = new ArrayList(); for (SearchHit hit : hits.getHits()) { Map<String, SearchHitField> fields = hit.getFields(); columns.add(fields.get(columnName).getValue()); } return columns; }
@Override public Object[] findIdsByColumn(String schemaName, String tableName, String pKeyName, String columnName, Object columnValue, Class entityClazz) { TermQueryBuilder filter = QueryBuilders.termQuery(columnName, columnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()).setTypes(tableName) .addField(pKeyName).setPostFilter(filter).execute().actionGet(); SearchHits hits = response.getHits(); Long length = hits.getTotalHits(); int absoluteLength = length.intValue(); Object[] ids = new Object[absoluteLength]; int counter = 0; for (SearchHit hit : hits.getHits()) { Map<String, SearchHitField> fields = hit.getFields(); ids[counter++] = fields.get(pKeyName).getValue(); } return ids; }
/** * Adds the fields to builder * * @param fieldsToSelect * the fields to select * @param clazz * the clazz * @param metaModel * the meta model * @param builder * the builder */ private void addFieldsToBuilder(String[] fieldsToSelect, Class clazz, MetamodelImpl metaModel, SearchRequestBuilder builder) { if (fieldsToSelect != null && fieldsToSelect.length > 1 && !(fieldsToSelect[1] == null)) { logger.debug("Fields added in query are: "); for (int i = 1; i < fieldsToSelect.length; i++) { logger.debug(i + " : " + fieldsToSelect[i]); builder = builder.addField(((AbstractAttribute) metaModel.entity(clazz).getAttribute(fieldsToSelect[i])) .getJPAColumnName()); } } }
public SearchRequestBuilder createScrollSearch(String oldIndexName, String oldType, String filter, int hitsPerPage, boolean withVersion, int keepTimeInMinutes, Client client) { SearchRequestBuilder srb = client.prepareSearch(oldIndexName). setTypes(oldType). setVersion(withVersion). setSize(hitsPerPage). setSearchType(SearchType.SCAN). addField("_source"). addField("_parent"). setScroll(TimeValue.timeValueMinutes(keepTimeInMinutes)); if (filter != null && !filter.trim().isEmpty()) srb.setPostFilter(filter); return srb; }
@Override public <E> List<E> getColumnsById(String schemaName, String tableName, String pKeyColumnName, String columnName, Object pKeyColumnValue, Class columnJavaType) { // fetch list ADDRESS_ID for given PERSON_ID QueryBuilder filterBuilder = new TermQueryBuilder(pKeyColumnName, pKeyColumnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()).setTypes(tableName) .setPostFilter(filterBuilder).addField(columnName).execute().actionGet(); SearchHits hits = response.getHits(); List columns = new ArrayList(); for (SearchHit hit : hits.getHits()) { Map<String, SearchHitField> fields = hit.getFields(); columns.add(fields.get(columnName).getValue()); } return columns; }
@Override public <E> List<E> getColumnsById(String schemaName, String tableName, String pKeyColumnName, String columnName, Object pKeyColumnValue, Class columnJavaType) { // fetch list ADDRESS_ID for given PERSON_ID FilterBuilder filterBuilder = new TermFilterBuilder(pKeyColumnName,pKeyColumnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()).setTypes(tableName).setFilter(filterBuilder).addField(columnName) .execute().actionGet(); SearchHits hits = response.getHits(); List columns = new ArrayList(); for (SearchHit hit : hits.getHits()) { Map<String,SearchHitField> fields = hit.getFields(); columns.add(fields.get(columnName).getValue()); } return columns; }
@Override public Object[] findIdsByColumn(String schemaName, String tableName, String pKeyName, String columnName, Object columnValue, Class entityClazz) { TermQueryBuilder filter = QueryBuilders.termQuery(columnName, columnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()).setTypes(tableName) .addField(pKeyName).setPostFilter(filter).execute().actionGet(); SearchHits hits = response.getHits(); Long length = hits.getTotalHits(); int absoluteLength = length.intValue(); Object[] ids = new Object[absoluteLength]; int counter = 0; for (SearchHit hit : hits.getHits()) { Map<String, SearchHitField> fields = hit.getFields(); ids[counter++] = fields.get(pKeyName).getValue(); } return ids; }
@Override public Object[] findIdsByColumn(String schemaName, String tableName, String pKeyName, String columnName, Object columnValue, Class entityClazz) { TermFilterBuilder filter = FilterBuilders.termFilter(columnName, columnValue); SearchResponse response = txClient.prepareSearch(schemaName.toLowerCase()) .setTypes(tableName).addField(pKeyName).setFilter(filter).execute().actionGet(); SearchHits hits = response.getHits(); Long length = hits.getTotalHits(); int absoluteLength = length.intValue(); Object[] ids = new Object[absoluteLength]; int counter=0; for (SearchHit hit : hits.getHits()) { Map<String,SearchHitField> fields = hit.getFields(); ids[counter++]= fields.get(pKeyName).getValue(); } return ids; }
@Override public void buildSearchForIndexedDocumentsNotUpdatedAfter(SearchRequestBuilder srb, String jiraProjectKey, Date date) { FilterBuilder filterTime = FilterBuilders.rangeFilter("_timestamp").lt(date); FilterBuilder filterProject = FilterBuilders.termFilter(indexFieldForProjectKey, jiraProjectKey); FilterBuilder filterSource = FilterBuilders.termFilter(indexFieldForRiverName, riverName); FilterBuilder filter = FilterBuilders.boolFilter().must(filterTime).must(filterProject).must(filterSource); srb.setQuery(QueryBuilders.matchAllQuery()).addField("_id").setPostFilter(filter); Set<String> st = new LinkedHashSet<String>(); st.add(issueTypeName); if (commentIndexingMode.isExtraDocumentIndexed()) st.add(commentTypeName); if (changelogIndexingMode.isExtraDocumentIndexed()) st.add(changelogTypeName); srb.setTypes(st.toArray(new String[st.size()])); }
private SearchRequestBuilder buildQuery(EnumSet<ElasticsearchDocumentType> elementType, boolean includeAggregations) { if (QUERY_LOGGER.isTraceEnabled()) { QUERY_LOGGER.trace("searching for: " + toString()); } List<FilterBuilder> filters = getFilters(elementType); QueryBuilder query = createQuery(getParameters()); query = scoringStrategy.updateQuery(query); AndFilterBuilder filterBuilder = getFilterBuilder(filters); String[] indicesToQuery = getIndexSelectionStrategy().getIndicesToQuery(this, elementType); if (QUERY_LOGGER.isTraceEnabled()) { QUERY_LOGGER.trace("indicesToQuery: %s", Joiner.on(", ").join(indicesToQuery)); } SearchRequestBuilder searchRequestBuilder = getClient() .prepareSearch(indicesToQuery) .setTypes(ElasticsearchSingleDocumentSearchIndex.ELEMENT_TYPE) .setQuery(QueryBuilders.filteredQuery(query, filterBuilder)) .addField(ElasticsearchSingleDocumentSearchIndex.ELEMENT_TYPE_FIELD_NAME) .addField(EXTENDED_DATA_ELEMENT_ID_FIELD_NAME) .addField(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_TABLE_NAME_FIELD_NAME) .addField(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_TABLE_ROW_ID_FIELD_NAME); if (includeAggregations) { List<AbstractAggregationBuilder> aggs = getElasticsearchAggregations(getAggregations()); for (AbstractAggregationBuilder aggregationBuilder : aggs) { searchRequestBuilder.addAggregation(aggregationBuilder); } } applySort(searchRequestBuilder); return searchRequestBuilder; }
private void deleteExtendedDataForElement(Element element) { try { FilterBuilder filter = FilterBuilders.termFilter(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_ELEMENT_ID_FIELD_NAME, element.getId()); SearchRequestBuilder s = getClient().prepareSearch(getIndicesToQuery()) .setTypes(ElasticsearchSingleDocumentSearchIndex.ELEMENT_TYPE) .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)) .addField(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_ELEMENT_ID_FIELD_NAME) .addField(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_TABLE_NAME_FIELD_NAME) .addField(ElasticsearchSingleDocumentSearchIndex.EXTENDED_DATA_TABLE_ROW_ID_FIELD_NAME); for (SearchHit hit : s.execute().get().getHits()) { if (MUTATION_LOGGER.isTraceEnabled()) { LOGGER.trace("deleting extended data document %s", hit.getId()); } getClient().prepareDelete(hit.getIndex(), ELEMENT_TYPE, hit.getId()).execute().actionGet(); } } catch (Exception ex) { throw new VertexiumException("Could not delete extended data for element: " + element.getId()); } }
private SearchRequestBuilder buildQuery(EnumSet<ElasticsearchDocumentType> elementType, boolean includeAggregations) { if (QUERY_LOGGER.isTraceEnabled()) { QUERY_LOGGER.trace("searching for: " + toString()); } List<QueryBuilder> filters = getFilters(elementType); QueryBuilder query = createQuery(getParameters()); query = scoringStrategy.updateQuery(query); QueryBuilder filterBuilder = getFilterBuilder(filters); String[] indicesToQuery = getIndexSelectionStrategy().getIndicesToQuery(this, elementType); if (QUERY_LOGGER.isTraceEnabled()) { QUERY_LOGGER.trace("indicesToQuery: %s", Joiner.on(", ").join(indicesToQuery)); } SearchRequestBuilder searchRequestBuilder = getClient() .prepareSearch(indicesToQuery) .setTypes(Elasticsearch2SearchIndex.ELEMENT_TYPE) .setQuery(QueryBuilders.boolQuery().must(query).filter(filterBuilder)) .addField(Elasticsearch2SearchIndex.ELEMENT_TYPE_FIELD_NAME) .addField(Elasticsearch2SearchIndex.EXTENDED_DATA_ELEMENT_ID_FIELD_NAME) .addField(Elasticsearch2SearchIndex.EXTENDED_DATA_TABLE_NAME_FIELD_NAME) .addField(Elasticsearch2SearchIndex.EXTENDED_DATA_TABLE_ROW_ID_FIELD_NAME); if (includeAggregations) { List<AbstractAggregationBuilder> aggs = getElasticsearchAggregations(getAggregations()); for (AbstractAggregationBuilder aggregationBuilder : aggs) { searchRequestBuilder.addAggregation(aggregationBuilder); } } applySort(searchRequestBuilder); return searchRequestBuilder; }
@Override public Set<String> getFolderNames() throws IOException, MessagingException { createIndexIfNotExists(); client.admin().indices().refresh(new RefreshRequest()).actionGet(); final HashSet<String> uids = new HashSet<String>(); SearchResponse scrollResp = client.prepareSearch().setIndices(index).setTypes(type).setSearchType(SearchType.SCAN) .setQuery(QueryBuilders.matchAllQuery()).addField("folderFullName").setScroll(new TimeValue(1000)).setSize(1000).execute() .actionGet(); while (true) { scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(1000)).execute().actionGet(); boolean hitsRead = false; for (final SearchHit hit : scrollResp.getHits()) { hitsRead = true; uids.add((String) hit.getFields().get("folderFullName").getValue()); } if (!hitsRead) { break; } } if (logger.isDebugEnabled()) { logger.debug("Currently locally stored folders: {}", uids); } return uids; }
private void deleteExtendedDataForElement(Element element) { try { QueryBuilder filter = QueryBuilders.termQuery(EXTENDED_DATA_ELEMENT_ID_FIELD_NAME, element.getId()); SearchRequestBuilder s = getClient().prepareSearch(getIndicesToQuery()) .setTypes(ELEMENT_TYPE) .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(filter)) .addField(EXTENDED_DATA_ELEMENT_ID_FIELD_NAME) .addField(EXTENDED_DATA_TABLE_NAME_FIELD_NAME) .addField(EXTENDED_DATA_TABLE_ROW_ID_FIELD_NAME); for (SearchHit hit : s.execute().get().getHits()) { if (MUTATION_LOGGER.isTraceEnabled()) { LOGGER.trace("deleting extended data document %s", hit.getId()); } getClient().prepareDelete(hit.getIndex(), ELEMENT_TYPE, hit.getId()).execute().actionGet(); } } catch (Exception ex) { throw new VertexiumException("Could not delete extended data for element: " + element.getId()); } }
.setQuery(QueryBuilders.matchAllQuery()).addField(itemIdField) .addSort(itemIdField, SortOrder.DESC).setSize(1) .execute(on(responseListener, failureListener));
FilterBuilders.termFilter(ProjectIndexingInfo.DOCFIELD_RIVER_NAME, riverName().getName()))) .setQuery(QueryBuilders.matchAllQuery()).addSort(ProjectIndexingInfo.DOCFIELD_START_DATE, SortOrder.DESC) .addField("_source").setSize(1).execute().actionGet(); if (sr.getHits().getTotalHits() > 0) { SearchHit hit = sr.getHits().getAt(0);
.setScroll(new TimeValue(keepAlive.longValue())) .setQuery(QueryBuilders.queryStringQuery(userQuery)) .addField(fieldName).setSize(size.intValue()).execute() .actionGet(); } else {
@Override protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) { final RiverConfigManager riverConfigManager = SingletonLaContainer.getComponent(RiverConfigManager.class); final RiverConfig riverConfig = riverConfigManager.get(crawlerContext.getSessionId()); if (riverConfig.isIncremental()) { final EsClient esClient = SingletonLaContainer.getComponent(EsClient.class); try { final SearchResponse response = esClient.prepareSearch(riverConfig.getIndex()).setTypes(riverConfig.getType()) .setQuery(QueryBuilders.termQuery("url", urlQueue.getUrl())).addField("lastModified") .addSort("lastModified", SortOrder.DESC).execute().actionGet(); final SearchHits hits = response.getHits(); if (hits.getTotalHits() > 0) { final SearchHitField lastModifiedField = hits.getAt(0).getFields().get("lastModified"); if (lastModifiedField != null) { final Date lastModified = ConversionUtil.convert(lastModifiedField.getValue(), Date.class); if (lastModified != null) { urlQueue.setLastModified(lastModified.getTime()); } } } } catch (final Exception e) { logger.debug("Failed to retrieve lastModified.", e); } } return super.isContentUpdated(client, urlQueue); } }