@Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { try { String groupNames = groupRequest.getGroups().stream().map(Group::getField).collect( Collectors.joining(",")); SolrQuery query = new SolrQuery() .setStart(0) .setRows(0) .setQuery(groupRequest.getQuery()); query.set("collection", getCollections(groupRequest.getIndices())); Optional<String> scoreField = groupRequest.getScoreField(); if (scoreField.isPresent()) { query.set("stats", true); query.set("stats.field", String.format("{!tag=piv1 sum=true}%s", scoreField.get())); } query.set("facet", true); query.set("facet.pivot", String.format("{!stats=piv1}%s", groupNames)); QueryResponse response = client.query(query); return buildGroupResponse(groupRequest, response); } catch (IOException | SolrServerException e) { String msg = e.getMessage(); LOG.error(msg, e); throw new InvalidSearchException(msg, e); } }
@Override public GroupResponse group(GroupRequest groupRequest) throws RestException { try { if (groupRequest.getIndices() == null || groupRequest.getIndices().isEmpty()) { groupRequest.setIndices(getDefaultIndices()); } return dao.group(groupRequest); } catch(InvalidSearchException ise) { throw new RestException(ise.getMessage(), ise); } }
@Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { // Make sure to escape any problematic characters here String sourceType = ClientUtils.escapeQueryChars(config.getSourceTypeField()); String baseQuery = groupRequest.getQuery(); String adjustedQuery = baseQuery + " -" + MetaAlertConstants.METAALERT_FIELD + ":[* TO *]" + " -" + sourceType + ":" + MetaAlertConstants.METAALERT_TYPE; LOG.debug("MetaAlert group adjusted query: {}", adjustedQuery); groupRequest.setQuery(adjustedQuery); return solrSearchDao.group(groupRequest); } }
/** * Build a group response. * @param groupRequest The original group request. * @param response The search response. * @return A group response. * @throws InvalidSearchException */ private GroupResponse buildGroupResponse( GroupRequest groupRequest, org.elasticsearch.action.search.SearchResponse response) throws InvalidSearchException { // build the search response Map<String, FieldType> commonColumnMetadata; try { commonColumnMetadata = columnMetadataDao.getColumnMetadata(groupRequest.getIndices()); } catch (IOException e) { throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(groupRequest.getIndices().toArray()))); } GroupResponse groupResponse = new GroupResponse(); groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); groupResponse.setGroupResults(getGroupResults(groupRequest, 0, response.getAggregations(), commonColumnMetadata)); return groupResponse; }
protected List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, List<PivotField> pivotFields) { List<Group> groups = groupRequest.getGroups(); List<GroupResult> searchResultGroups = new ArrayList<>(); final GroupOrder groupOrder = groups.get(index).getOrder(); groupResult.setKey(pivotField.getValue().toString()); groupResult.setTotal(pivotField.getCount()); Optional<String> scoreField = groupRequest.getScoreField(); if (scoreField.isPresent()) { groupResult
/** * Build a group response. * @param groupRequest The original group request. * @param response The search response. * @return A group response. */ protected GroupResponse buildGroupResponse( GroupRequest groupRequest, QueryResponse response) { String groupNames = groupRequest.getGroups().stream().map(Group::getField).collect( Collectors.joining(",")); List<PivotField> pivotFields = response.getFacetPivot().get(groupNames); GroupResponse groupResponse = new GroupResponse(); groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); groupResponse.setGroupResults(getGroupResults(groupRequest, 0, pivotFields)); return groupResponse; }
/** * Builds a group search request. * @param groupRequest The Metron group request. * @param queryBuilder The search query. * @return An Elasticsearch search request. */ private org.elasticsearch.action.search.SearchRequest buildGroupRequest( GroupRequest groupRequest, QueryBuilder queryBuilder) { // handle groups TermsAggregationBuilder groups = getGroupsTermBuilder(groupRequest, 0); final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .query(queryBuilder) .aggregation(groups); // return the search request String[] indices = wildcardIndices(groupRequest.getIndices()); return new org.elasticsearch.action.search.SearchRequest() .indices(indices) .source(searchSourceBuilder); }
@Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { return group(groupRequest, new QueryStringQueryBuilder(groupRequest.getQuery())); }
private TermsAggregationBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) { List<Group> groups = groupRequest.getGroups(); Group group = groups.get(index); String aggregationName = getGroupByAggregationName(group.getField()); TermsAggregationBuilder termsBuilder = AggregationBuilders.terms(aggregationName); termsBuilder .field(group.getField()) .size(accessConfig.getMaxSearchGroups()) .order(getElasticsearchGroupOrder(group.getOrder())); if (index < groups.size() - 1) { termsBuilder.subAggregation(getGroupsTermBuilder(groupRequest, index + 1)); } Optional<String> scoreField = groupRequest.getScoreField(); if (scoreField.isPresent()) { SumAggregationBuilder scoreSumAggregationBuilder = AggregationBuilders.sum(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0); termsBuilder.subAggregation(scoreSumAggregationBuilder); } return termsBuilder; }
/** * Defers to a provided {@link org.elasticsearch.index.query.QueryBuilder} for the query. * @param groupRequest The request defining the parameters of the grouping * @param queryBuilder The actual query to be run. Intended for if the SearchRequest requires wrapping * @return The results of the query * @throws InvalidSearchException When the query is malformed or the current state doesn't allow search */ protected GroupResponse group(GroupRequest groupRequest, QueryBuilder queryBuilder) throws InvalidSearchException { org.elasticsearch.action.search.SearchRequest esRequest; org.elasticsearch.action.search.SearchResponse esResponse; if (client == null) { throw new InvalidSearchException("Uninitialized Dao! You must call init() prior to use."); } if (groupRequest.getGroups() == null || groupRequest.getGroups().size() == 0) { throw new InvalidSearchException("At least 1 group must be provided."); } esRequest = buildGroupRequest(groupRequest, queryBuilder); esResponse = requestSubmitter.submitSearch(esRequest); GroupResponse response = buildGroupResponse(groupRequest, esResponse); return response; }
@Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { // Wrap the query to hide any alerts already contained in meta alerts QueryBuilder qb = QueryBuilders.boolQuery() .must(new QueryStringQueryBuilder(groupRequest.getQuery())) .mustNot(existsQuery(MetaAlertConstants.METAALERT_FIELD)); return elasticsearchDao.group(groupRequest, qb); }
private List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, Aggregations aggregations, Map<String, FieldType> commonColumnMetadata) { List<Group> groups = groupRequest.getGroups(); String field = groups.get(index).getField(); List<GroupResult> searchResultGroups = new ArrayList<>(); if(aggregations != null) { Terms terms = aggregations.get(getGroupByAggregationName(field)); for (Bucket bucket : terms.getBuckets()) { GroupResult groupResult = new GroupResult(); groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); groupResult.setTotal(bucket.getDocCount()); Optional<String> scoreField = groupRequest.getScoreField(); if (scoreField.isPresent()) { Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); groupResult.setScore(score.getValue()); } if (index < groups.size() - 1) { groupResult.setGroupedBy(groups.get(index + 1).getField()); groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); } searchResultGroups.add(groupResult); } } return searchResultGroups; } }
@Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { GroupResponse groupResponse = new GroupResponse(); groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); groupResponse.setGroupResults(getGroupResults(groupRequest.getGroups(), 0)); return groupResponse; }