@Override public Aggregation getAggregation(String name) { return aggregations == null ? null : aggregations.get(name); }
private void processMissingAggregation(Missing aggregation) { long docCount = aggregation.getDocCount(); if (docCount > 0L) { LinkedHashMap<String, Long> facet = getOrCreateFacet(aggregation.getName().replace("_missing", "")); if (aggregation.getAggregations().getAsMap().containsKey(FACET_MODE_EFFORT)) { facet.put("", Math.round(((Sum) aggregation.getAggregations().get(FACET_MODE_EFFORT)).getValue())); } else { facet.put("", docCount); } } }
private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) { InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); List<InternalBucket> buckets = filtersAgg.getBuckets(); return ComponentIndexResults.newBuilder() .setQualifiers( buckets.stream().map(ComponentIndex::bucketToQualifier)) .build(); }
public SearchResult(SearchResponse resp, Select select) throws SqlParseException { Aggregations aggs = resp.getAggregations(); if (aggs.get("filter") != null) { InternalFilter inf = aggs.get("filter"); aggs = inf.getAggregations(); } if (aggs.get("group by") != null) { InternalTerms terms = aggs.get("group by"); Collection<Bucket> buckets = terms.getBuckets(); this.total = buckets.size(); results = new ArrayList<>(buckets.size()); for (Bucket bucket : buckets) { Map<String, Object> aggsMap = toAggsMap(bucket.getAggregations().getAsMap()); aggsMap.put("docCount", bucket.getDocCount()); results.add(aggsMap); } } else { results = new ArrayList<>(1); this.total = 1; Map<String, Object> map = new HashMap<>(); for (Aggregation aggregation : aggs) { map.put(aggregation.getName(), covenValue(aggregation)); } results.add(map); } }
/** * Get result for time limited aggregation for all countries and required classes. * @param aggr Range aggregation object * @param classes Classes to consider * @return HashMap with parsed results */ private static HashMap<String, HashMap<String, HashMap<String, Double>>> getAggregationByTimeWithCountries(Range aggr, List<String> classes) { HashMap<String, HashMap<String, HashMap<String, Double>>> retMap = new HashMap<>(); for (Range.Bucket bucket : aggr.getBuckets()) { retMap = getAggregationByCountry(bucket.getAggregations().get("by_country"), classes); } return retMap; }
private void processDateHistogram(Histogram aggregation) { LinkedHashMap<String, Long> facet = getOrCreateFacet(aggregation.getName()); for (Histogram.Bucket value : aggregation.getBuckets()) { String day = dateTimeToDate(value.getKeyAsString(), timeZone); if (value.getAggregations().getAsMap().containsKey(FACET_MODE_EFFORT)) { facet.put(day, Math.round(((Sum) value.getAggregations().get(FACET_MODE_EFFORT)).getValue())); } else { facet.put(day, value.getDocCount()); } } }
/** * Get result for time limited aggregation for all available classes. * @param aggr Range aggregation object * @param classes Classes to consider * @return HashMap with parsed aggregations */ private static HashMap<String, HashMap<String, Double>> getAggregationByTime(Range aggr, List<String> classes) { HashMap<String, HashMap<String, Double>> retMap = new HashMap<>(); for (Range.Bucket bucket : aggr.getBuckets()) { retMap = getAggregationByClass(bucket.getAggregations().get("by_class"), classes); } return retMap; }
private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, @Nullable List<SecurityStandardCategoryStatistics> children) { List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get("vulnerabilities")).getAggregations().get("severity")) .getBuckets(); long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue()).sum(); // Worst severity having at least one issue OptionalInt severityRating = severityBuckets.stream() .filter(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue() != 0) .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) .max(); long openSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("openSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("toReviewSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long wontFixSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("wontFixSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); return new SecurityStandardCategoryStatistics(categoryName, vulnerabilities, severityRating, toReviewSecurityHotspots, openSecurityHotspots, wontFixSecurityHotspots, children); }
/** * Get aggregation result for all countries. * @param aggr Terms aggregation object * @param classes Classes to consider * @return HashMap with parsed aggregations */ private static HashMap<String, HashMap<String, HashMap<String, Double>>> getAggregationByCountry(Terms aggr, List<String> classes) { HashMap<String, HashMap<String, HashMap<String, Double>>> retMap = new HashMap<>(); for (Terms.Bucket bucket : aggr.getBuckets()) { retMap.put(bucket.getKeyAsString(), getAggregationByClass(bucket.getAggregations().get("by_class"), classes)); } return retMap; }
/** * Get all time aggregations without any geographical constraints. * @param index Name of ES index * @param classifierName Classifier for aggregation * @param classes Classes to consider * @return HashMap with required aggregations */ private final HashMap<String, HashMap<String, Double>> classifierScore(String index, String classifierName, List<String> classes) { SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilder(classifierName)); Terms aggrs = response.getAggregations().get("by_class"); return getAggregationByClass(aggrs, classes); }
private long getEntitySizeAggregation( final SearchRequestBuilder builder ) { final String key = "entitySize"; SumBuilder sumBuilder = new SumBuilder(key); sumBuilder.field("entitySize"); builder.addAggregation(sumBuilder); Observable<Number> o = Observable.from(builder.execute()) .map(response -> { Sum aggregation = (Sum) response.getAggregations().get(key); if(aggregation == null){ return -1; }else{ return aggregation.getValue(); } }); Number val = ObservableTimer.time(o,aggregationTimer).toBlocking().lastOrDefault(-1); return val.longValue(); }
/** * Get all time aggregation by country for all available countries. * @param index Name of ES index * @param classifierName Name of classifier * @param classes Classes to consider * @return HashMap with required aggregations */ private final HashMap<String, HashMap<String, HashMap<String,Double>>> classifierScoreForCountry(String index, String classifierName, List<String> classes) { SearchResponse response = getAggregationResponse(index, getClassifierAggregationBuilderByCountry("place_country_code", classifierName)); Terms aggr = response.getAggregations().get("by_country"); HashMap<String, HashMap<String, HashMap<String,Double>>> retMap = getAggregationByCountry(aggr, classes); // Put global score retMap.put("GLOBAL", classifierScore(index, classifierName, classes)); return retMap; }
private OptionalLong getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String facetNameAndField = CREATED_AT.getFieldName(); SearchRequestBuilder esRequest = client .prepareSearch(INDEX_TYPE_ISSUE) .setSize(0); BoolQueryBuilder esFilter = boolQuery(); filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { esRequest.setQuery(QueryBuilders.boolQuery().must(esQuery).filter(esFilter)); } else { esRequest.setQuery(esQuery); } esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); Min minValue = esRequest.get().getAggregations().get(facetNameAndField); double actualValue = minValue.getValue(); if (Double.isInfinite(actualValue)) { return OptionalLong.empty(); } return OptionalLong.of((long)actualValue); }
public List<BranchStatistics> searchBranchStatistics(String projectUuid, List<String> branchUuids) { if (branchUuids.isEmpty()) { return Collections.emptyList(); } SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE) .setRouting(projectUuid) .setQuery( boolQuery() .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .must(termQuery(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(false)))) .setSize(0) .addAggregation(AggregationBuilders.terms("branchUuids") .field(FIELD_ISSUE_BRANCH_UUID) .size(branchUuids.size()) .subAggregation(AggregationBuilders.terms("types") .field(FIELD_ISSUE_TYPE))); SearchResponse response = request.get(); return ((StringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() .map(bucket -> new BranchStatistics(bucket.getKeyAsString(), ((StringTerms) bucket.getAggregations().get("types")).getBuckets() .stream() .collect(uniqueIndex(StringTerms.Bucket::getKeyAsString, InternalTerms.Bucket::getDocCount)))) .collect(MoreCollectors.toList(branchUuids.size())); }
private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) { InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); SearchHits hitList = docs.getHits(); SearchHit[] hits = hitList.getHits(); return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits()); }
private void processHistogramAggregation(Histogram agg) { List<IntervalUnit> intervals = new ArrayList<>(); for (Histogram.Bucket h : agg.getBuckets()) { ExtendedStats hStats = h.getAggregations().get(AbstractFacetRequest.INTERNAL_STATS); if (hStats != null) { intervals.add(new IntervalUnit(((DateTime) h.getKey()).getMillis(), h.getDocCount(), h.getDocCount(), hStats.getSum(), hStats.getAvg(), hStats.getMin(), hStats.getMax())); } else { intervals.add(new IntervalUnit(((DateTime) h.getKey()).getMillis(), h.getDocCount(), h.getDocCount(), 0, 0, 0, 0)); } } addFacet(new HistogramResult(agg.getName(), intervals)); } }
private void processRangeAggregation(Range agg) { List<? extends Range.Bucket> buckets = ((Range) agg).getBuckets(); List<org.springframework.data.elasticsearch.core.facet.result.Range> ranges = new ArrayList<>(); for (Range.Bucket b : buckets) { ExtendedStats rStats = b.getAggregations().get(AbstractFacetRequest.INTERNAL_STATS); if (rStats != null) { Sum sum = b.getAggregations().get(RangeFacetRequest.RANGE_INTERNAL_SUM); ranges.add(new org.springframework.data.elasticsearch.core.facet.result.Range((Double) b.getFrom(), (Double) b.getTo(), b.getDocCount(), sum != null ? sum.getValue() : rStats.getSum(), rStats.getCount(), rStats.getMin(), rStats.getMax())); } else { ranges.add(new org.springframework.data.elasticsearch.core.facet.result.Range((Double) b.getFrom(), (Double) b.getTo(), b.getDocCount(), 0, 0, 0, 0)); } } addFacet(new RangeResult(agg.getName(), ranges)); }
public List<String> searchTags(@Nullable String textQuery, int size) { int maxPageSize = 500; checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize); if (size <= 0) { return emptyList(); } TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) .field(FIELD_TAGS) .size(size) .minDocCount(1) .order(Terms.Order.term(true)); if (textQuery != null) { tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } SearchRequestBuilder searchQuery = client .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) .setQuery(authorizationTypeSupport.createQueryFilter()) .setFetchSource(false) .setSize(0) .addAggregation(tagFacet); Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); return aggregation.getBuckets().stream() .map(Bucket::getKeyAsString) .collect(MoreCollectors.toList()); }
private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, Terms.Order termsOrder, int size) { SearchRequestBuilder requestBuilder = client .prepareSearch(INDEX_TYPE_ISSUE) // Avoids returning search hits .setSize(0); requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); TermsAggregationBuilder aggreg = AggregationBuilders.terms("_ref") .field(fieldName) .size(size) .order(termsOrder) .minDocCount(1L); if (textQuery != null) { aggreg.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery)), null)); } SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); return searchResponse.getAggregations().get("_ref"); }
public List<String> terms(String fields, @Nullable String query, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) .field(fields) .size(size) .minDocCount(1); if (query != null) { termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); } SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) .setQuery(matchAllQuery()) .setSize(0) .addAggregation(termsAggregation); SearchResponse esResponse = request.get(); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME)); }