public SourceFieldsContext(SearchRequest request) { if (request.source() != null && request.source().fetchSource() != null) { includes = request.source().fetchSource().includes(); excludes = request.source().fetchSource().excludes(); fetchSource = request.source().fetchSource().fetchSource(); } //if (request.source() != null && request.source().storedFields() != null && request.source().storedFields().fieldNames() != null) { // storedFields = request.source().storedFields().fieldNames().toArray(new String[0]); //} }
public static boolean isNeeded(SearchRequest request) { return (request.source() != null && request.source().fetchSource() != null && (request.source().fetchSource().includes() != null || request .source().fetchSource().excludes() != null)) || (request.source() != null && request.source().storedFields() != null && request.source().storedFields().fieldNames() != null && !request.source().storedFields().fieldNames().isEmpty()); }
public SearchResponse suggest(SuggestBuilder suggestion, String... indices) { SearchRequest searchRequest = new SearchRequest(indices); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.suggest(suggestion); searchRequest.source(sourceBuilder); try { return client.search(searchRequest); } catch (IOException e) { throw new ElasticsearchException("Could not execute search request : " + searchRequest.toString(), e); } }
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.boolQuery().must(queryBuilder).filter(standardFilters(range, filter))); searchSourceBuilder.from(offset); searchSourceBuilder.size(limit); searchSourceBuilder.sort(sort.getField(), sort.asElastic()); .fragmentSize(0) .numOfFragments(0); searchSourceBuilder.highlighter(highlightBuilder);
private long totalCount(final String[] indexNames) { // Return 0 if there are no indices in the given index set. If we run the query with an empty index list, // Elasticsearch will count all documents in all indices and thus return a wrong count. if (indexNames.length == 0) { return 0L; } final List<String> indices = Arrays.asList(indexNames); final String query = new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) .size(0) .toString(); final Search request = new Search.Builder(query) .addIndex(indices) .build(); final MultiSearch multiSearch = new MultiSearch.Builder(request).build(); final MultiSearchResult searchResult = JestUtils.execute(jestClient, multiSearch, () -> "Fetching message count failed for indices " + indices); final List<MultiSearchResult.MultiSearchResponse> responses = searchResult.getResponses(); long total = 0L; for (MultiSearchResult.MultiSearchResponse response : responses) { if (response.isError) { throw JestUtils.specificException(() -> "Fetching message count failed for indices " + indices, response.error); } total += response.searchResult.getTotal(); } return total; } }
private SearchRequest prepareScroll(Query query, long scrollTimeInMillis) { SearchRequest request = new SearchRequest(toArray(query.getIndices())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); request.types(toArray(query.getTypes())); request.scroll(TimeValue.timeValueMillis(scrollTimeInMillis)); if (query.getPageable().isPaged()) { searchSourceBuilder.size(query.getPageable().getPageSize()); } if (!isEmpty(query.getFields())) { searchSourceBuilder.fetchSource(toArray(query.getFields()), null); } request.source(searchSourceBuilder); return request; }
private long doCount(SearchRequest countRequest, QueryBuilder elasticsearchQuery) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); if (elasticsearchQuery != null) { sourceBuilder.query(elasticsearchQuery); } countRequest.source(sourceBuilder); try { return client.search(countRequest).getHits().getTotalHits(); } catch (IOException e) { throw new ElasticsearchException("Error while searching for request: " + countRequest.toString(), e); } }
private long doCount(SearchRequest searchRequest, QueryBuilder elasticsearchQuery, QueryBuilder elasticsearchFilter) { if (elasticsearchQuery != null) { searchRequest.source().query(elasticsearchQuery); } else { searchRequest.source().query(QueryBuilders.matchAllQuery()); } if (elasticsearchFilter != null) { searchRequest.source().postFilter(elasticsearchFilter); } SearchResponse response; try { response = client.search(searchRequest); } catch (IOException e) { throw new ElasticsearchException("Error for search request: " + searchRequest.toString(), e); } return response.getHits().getTotalHits(); }
private SearchResponse doScroll(SearchRequest request, CriteriaQuery criteriaQuery) { Assert.notNull(criteriaQuery.getIndices(), "No index defined for Query"); Assert.notNull(criteriaQuery.getTypes(), "No type define for Query"); Assert.notNull(criteriaQuery.getPageable(), "Query.pageable is required for scan & scroll"); QueryBuilder elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria()); QueryBuilder elasticsearchFilter = new CriteriaFilterProcessor() .createFilterFromCriteria(criteriaQuery.getCriteria()); if (elasticsearchQuery != null) { request.source().query(elasticsearchQuery); } else { request.source().query(QueryBuilders.matchAllQuery()); } if (elasticsearchFilter != null) { request.source().postFilter(elasticsearchFilter); } request.source().version(true); try { return client.search(request); } catch (IOException e) { throw new ElasticsearchException("Error for search request with scroll: " + request.toString(), e); } }
@Override public <T> Page<T> queryForPage(StringQuery query, Class<T> clazz, SearchResultMapper mapper) { SearchRequest request = prepareSearch(query, clazz); request.source().query((wrapperQuery(query.getSource()))); SearchResponse response; try { response = client.search(request); } catch (IOException e) { throw new ElasticsearchException("Error for search request: " + request.toString(), e); } return mapper.mapResults(response, clazz, query.getPageable()); }
public HistogramResult histogram(String query, DateHistogramInterval interval, String filter, TimeRange range) { final DateHistogramAggregationBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM) .field(Message.FIELD_TIMESTAMP) .dateHistogramInterval(interval.toESInterval()); final SearchSourceBuilder searchSourceBuilder = filteredSearchRequest(query, filter, range) .aggregation(histogramBuilder); final Set<String> affectedIndices = determineAffectedIndices(range, filter); if (affectedIndices.isEmpty()) { return DateHistogramResult.empty(query, searchSourceBuilder.toString(), interval); } final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()) .addType(IndexMapping.TYPE_MESSAGE) .addIndex(affectedIndices) .ignoreUnavailable(true) .allowNoIndices(true); final io.searchbox.core.SearchResult searchResult = wrapInMultiSearch(searchBuilder.build(), () -> "Unable to retrieve histogram"); recordEsMetrics(searchResult, range); final HistogramAggregation histogramAggregation = searchResult.getAggregations().getHistogramAggregation(AGG_HISTOGRAM); return new DateHistogramResult( histogramAggregation, query, searchSourceBuilder.toString(), interval, tookMsFromSearchResult(searchResult) ); }
public CountResult count(String query, TimeRange range, String filter) { final Set<String> affectedIndices = determineAffectedIndices(range, filter); if (affectedIndices.isEmpty()) { return CountResult.empty(); } final String searchSource = standardSearchRequest(query, 0, -1, range, filter, null, false).toString(); final Search search = new Search.Builder(searchSource).addIndex(affectedIndices).build(); final io.searchbox.core.SearchResult searchResult = wrapInMultiSearch(search, () -> "Unable to perform count query"); recordEsMetrics(searchResult, range); return CountResult.create(searchResult.getTotal(), 0); }
private SearchResponse doScroll(SearchRequest request, SearchQuery searchQuery) { Assert.notNull(searchQuery.getIndices(), "No index defined for Query"); Assert.notNull(searchQuery.getTypes(), "No type define for Query"); Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll"); if (searchQuery.getFilter() != null) { request.source().postFilter(searchQuery.getFilter()); } request.source().version(true); try { return client.search(request); } catch (IOException e) { throw new ElasticsearchException("Error for search request with scroll: " + request.toString(), e); } }
@Override public String explain() { try { XContentBuilder firstBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); this.firstSearchRequest.request().source().toXContent(firstBuilder, ToXContent.EMPTY_PARAMS); XContentBuilder secondBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); this.secondSearchRequest.request().source().toXContent(secondBuilder, ToXContent.EMPTY_PARAMS); String explained = String.format("performing %s on :\n left query:\n%s\n right query:\n%s", this.relation.name, BytesReference.bytes(firstBuilder).utf8ToString(), BytesReference.bytes(secondBuilder).utf8ToString()); return explained; } catch (IOException e) { e.printStackTrace(); } return null; }
@Override public <T> List<String> queryForIds(SearchQuery query) { SearchRequest request = prepareSearch(query, Optional.ofNullable(query.getQuery())); request.source().query(query.getQuery()); if (query.getFilter() != null) { request.source().postFilter(query.getFilter()); } SearchResponse response; try { response = client.search(request); } catch (IOException e) { throw new ElasticsearchException("Error for search request: " + request.toString(), e); } return extractIds(response); }
public ScrollResult scroll(String query, TimeRange range, int limit, int offset, List<String> fields, String filter) { final Set<String> affectedIndices = determineAffectedIndices(range, filter); final Set<String> indexWildcards = indexSetRegistry.getForIndices(affectedIndices).stream() .map(IndexSet::getIndexWildcard) .collect(Collectors.toSet()); final String searchQuery; final Sorting sorting = new Sorting("_doc", Sorting.Direction.ASC); if (filter == null) { searchQuery = standardSearchRequest(query, limit, offset, range, sorting).toString(); } else { searchQuery = filteredSearchRequest(query, filter, limit, offset, range, sorting).toString(); } final Search.Builder initialSearchBuilder = new Search.Builder(searchQuery) .addType(IndexMapping.TYPE_MESSAGE) .setParameter(Parameters.SCROLL, "1m") .addIndex(indexWildcards); fields.forEach(initialSearchBuilder::addSourceIncludePattern); final io.searchbox.core.SearchResult initialResult = checkForFailedShards(JestUtils.execute(jestClient, initialSearchBuilder.build(), () -> "Unable to perform scroll search")); recordEsMetrics(initialResult, range); return scrollResultFactory.create(initialResult, query, fields); }
private SearchSourceBuilder searchRequest(SearchesConfig config) { final SearchSourceBuilder request; if (config.filter() == null) { request = standardSearchRequest(config.query(), config.limit(), config.offset(), config.range(), config.sorting()); } else { request = filteredSearchRequest(config.query(), config.filter(), config.limit(), config.offset(), config.range(), config.sorting()); } final List<String> fields = config.fields(); if (fields != null) { // Use source filtering instead of SearchSourceBuilder#fields() here because Jest cannot handle responses // without a "_source" field yet. See: // https://github.com/searchbox-io/Jest/issues/157 // https://github.com/searchbox-io/Jest/issues/339 request.fetchSource(fields.toArray(new String[fields.size()]), Strings.EMPTY_ARRAY); } return request; }
@Override public String explain() { try { XContentBuilder firstBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); firstTable.getRequestBuilder().request().source().toXContent(firstBuilder, ToXContent.EMPTY_PARAMS); XContentBuilder secondBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); secondTable.getRequestBuilder().request().source().toXContent(secondBuilder, ToXContent.EMPTY_PARAMS); String explained = String.format(" first query:\n%s\n second query:\n%s", BytesReference.bytes(firstBuilder).utf8ToString(), BytesReference.bytes(secondBuilder).utf8ToString()); return explained; } catch (IOException e) { e.printStackTrace(); } return null; }
@SuppressWarnings("unchecked") public SearchResult search(SearchesConfig config) { final Set<IndexRange> indexRanges = determineAffectedIndicesWithRanges(config.range(), config.filter()); final SearchSourceBuilder requestBuilder = searchRequest(config); if (indexRanges.isEmpty()) { return SearchResult.empty(config.query(), requestBuilder.toString()); } final Set<String> indices = extractIndexNamesFromIndexRanges(indexRanges); final Search.Builder searchBuilder = new Search.Builder(requestBuilder.toString()) .addType(IndexMapping.TYPE_MESSAGE) .addIndex(indices); final io.searchbox.core.SearchResult searchResult = wrapInMultiSearch(searchBuilder.build(), () -> "Unable to perform search query"); final List<ResultMessage> hits = searchResult.getHits(Map.class, false).stream() .map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)) .collect(Collectors.toList()); recordEsMetrics(searchResult, config.range()); return new SearchResult(hits, searchResult.getTotal(), indexRanges, config.query(), requestBuilder.toString(), tookMsFromSearchResult(searchResult)); }
public static String getSearchRequestAsString(SearchRequest request) { String query = "POST /"; if (request.indices().length > 0) { query += StringUtils.asCsv(request.indices()) + "/"; if (request.types().length > 0) { query += StringUtils.asCsv(request.types()) + "/"; } } query += "_search"; query += getQueryParameters(request); query += "\n"; query += request.source().toString(); return query; }