public static Map<String, Long> termsToMap(Terms terms) { LinkedHashMap<String, Long> map = new LinkedHashMap<>(); List<? extends Terms.Bucket> buckets = terms.getBuckets(); for (Terms.Bucket bucket : buckets) { map.put(bucket.getKeyAsString(), bucket.getDocCount()); } return map; }
private TermsAggregationBuilder buildTermsFacetAggregation(String fieldName, String facetName, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(facetName) .field(fieldName) .order(order) .size(size) .minDocCount(FACET_DEFAULT_MIN_DOC_COUNT); if (subAggregation != null) { termsAggregation = termsAggregation.subAggregation(subAggregation); } return termsAggregation; }
private void processTermsAggregation(Terms aggregation) { String facetName = aggregation.getName(); // TODO document this naming convention if (facetName.contains("__") && !facetName.startsWith("__")) { facetName = facetName.substring(0, facetName.indexOf("__")); } facetName = facetName.replace("_selected", ""); LinkedHashMap<String, Long> facet = getOrCreateFacet(facetName); for (Terms.Bucket value : aggregation.getBuckets()) { List<Aggregation> aggregationList = value.getAggregations().asList(); if (aggregationList.size() == 1) { facet.put(value.getKeyAsString(), Math.round(((Sum) aggregationList.get(0)).getValue())); } else { facet.put(value.getKeyAsString(), value.getDocCount()); } } }
private void processTermAggregation(Terms agg) { List<Term> terms = new ArrayList<>(); for (Terms.Bucket t : agg.getBuckets()) { terms.add(new Term(t.getKeyAsString(), t.getDocCount())); } addFacet(new TermResult(agg.getName(), terms, terms.size(), agg.getSumOfOtherDocCounts(), 0)); }
/** * Get aggregation result for all countries. * @param aggr Terms aggregation object * @param classes Classes to consider * @return HashMap with parsed aggregations */ private static HashMap<String, HashMap<String, HashMap<String, Double>>> getAggregationByCountry(Terms aggr, List<String> classes) { HashMap<String, HashMap<String, HashMap<String, Double>>> retMap = new HashMap<>(); for (Terms.Bucket bucket : aggr.getBuckets()) { retMap.put(bucket.getKeyAsString(), getAggregationByClass(bucket.getAggregations().get("by_class"), classes)); } return retMap; }
for(int i = 0; i<((StringTerms) brandNames).getBuckets().size(); i++){ brandNameList.add(((StringTerms) brandNames).getBuckets().get(i).getKeyAsString()); for(int i=0;i<((StringTerms) productCategoryNames).getBuckets().size();i++){ productCategoryNameList.add(((StringTerms) productCategoryNames).getBuckets().get(i).getKeyAsString()); List<Terms.Bucket> attrIds = ((LongTerms) ((InternalFilter)productAttrs.getProperty("productAttrs")).getAggregations().getProperty("attrIds")).getBuckets(); List<EsProductRelatedInfo.ProductAttr> attrList = new ArrayList<>(); for (Terms.Bucket attrId : attrIds) { EsProductRelatedInfo.ProductAttr attr = new EsProductRelatedInfo.ProductAttr(); attr.setAttrId((Long) attrId.getKey()); List<String> attrValueList = new ArrayList<>(); List<Terms.Bucket> attrValues = ((StringTerms) attrId.getAggregations().get("attrValues")).getBuckets(); List<Terms.Bucket> attrNames = ((StringTerms) attrId.getAggregations().get("attrNames")).getBuckets(); for (Terms.Bucket attrValue : attrValues) { attrValueList.add(attrValue.getKeyAsString()); String attrName = attrNames.get(0).getKeyAsString(); attr.setAttrName(attrName);
SearchResponse response = client().prepareSearch("idx").setTypes("type") .setQuery(matchAllQuery()) .addAggregation(terms("keys").field("key").size(3).order(Terms.Order.count(false))) .execute().actionGet(); Terms terms = response.getAggregations().get("keys"); Collection<Terms.Bucket> buckets = terms.getBuckets(); assertThat(buckets.size(), equalTo(3));
/** * Get AggregationBuilder by country without time constraints. * @param fieldName Name of field containing country code * @param classifierName Name of field containing country code * @return AggregationBuilder with required configuration */ private static TermsBuilder getClassifierAggregationBuilderByCountry(String fieldName, String classifierName) { return AggregationBuilders.terms("by_country").field(fieldName) .subAggregation(getClassifierAggregationBuilder(classifierName)); }
private static AggregationBuilder addSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe) { AggregationBuilder aggregationBuilder = addSecurityReportIssueCountAggregations(categoriesAggs); if (includeCwe) { categoriesAggs .subAggregation(addSecurityReportIssueCountAggregations(AggregationBuilders.terms("cwe").field(FIELD_ISSUE_CWE))); } return aggregationBuilder; }
private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, @Nullable List<SecurityStandardCategoryStatistics> children) { List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get("vulnerabilities")).getAggregations().get("severity")) .getBuckets(); long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue()).sum(); // Worst severity having at least one issue OptionalInt severityRating = severityBuckets.stream() .filter(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue() != 0) .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) .max(); long openSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("openSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("toReviewSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long wontFixSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("wontFixSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); return new SecurityStandardCategoryStatistics(categoryName, vulnerabilities, severityRating, toReviewSecurityHotspots, openSecurityHotspots, wontFixSecurityHotspots, children); }
private void setSize (AggregationBuilder agg, Field field) { if (field instanceof MethodField) { //zhongshu-comment MethodField可以自定义聚合的size MethodField mf = ((MethodField) field); Object customSize = mf.getParamsAsMap().get("size"); if (customSize == null) { //zhongshu-comment 假如用户没有在MethodField指定agg的size,就将默认的rowCount设置为agg的size if(select.getRowCount()>0) { if (agg instanceof TermsAggregationBuilder) { ((TermsAggregationBuilder) agg).size(select.getRowCount()); } } } else { //zhongshu-comment 不需要任何操作,因为之前步骤的代码已经将自定义的size设置到agg对象中了 } } else { if(select.getRowCount()>0) { if (agg instanceof TermsAggregationBuilder) { ((TermsAggregationBuilder) agg).size(select.getRowCount()); } } } }
private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(InternalFilter categoryBucket, boolean includeCwe) { List<SecurityStandardCategoryStatistics> children = new ArrayList<>(); if (includeCwe) { ((StringTerms) categoryBucket.getAggregations().get("cwe")).getBuckets() .forEach(cweBucket -> children.add(processSecurityReportCategorySearchResults(cweBucket, cweBucket.getKeyAsString(), null))); } return processSecurityReportCategorySearchResults(categoryBucket, categoryBucket.getName(), children); }
public List<String> searchAuthors(IssueQuery query, @Nullable String textQuery, int maxNumberOfAuthors) { Terms terms = listTermsMatching(FIELD_ISSUE_AUTHOR_LOGIN, query, textQuery, Terms.Order.term(true), maxNumberOfAuthors); return EsUtils.termsKeys(terms); }
public Map<String, Long> countTags(IssueQuery query, int maxNumberOfTags) { Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, null, Terms.Order.count(false), maxNumberOfTags); return EsUtils.termsToMap(terms); }
private void setShardSize(AggregationBuilder agg) { if (agg instanceof TermsAggregationBuilder) { int defaultShardSize = 20 * select.getRowCount(); ((TermsAggregationBuilder) agg).shardSize(Math.max(defaultShardSize, 5000)); } }
/** * Get aggregation results for each class as sum, avg and doc count. * @param aggr Terms aggregation object * @param classes Classes to be taken into account * @return HashMap with required aggregations */ private static HashMap<String, HashMap<String, Double>> getAggregationByClass(Terms aggr, List<String> classes) { HashMap<String, HashMap<String, Double>> retMap = new HashMap<>(); for (Terms.Bucket bucket : aggr.getBuckets()) { String key = bucket.getKeyAsString(); if (!classes.contains(key)) { continue; } long docCount = bucket.getDocCount(); Sum sum = bucket.getAggregations().get("sum_probability"); Avg avg = bucket.getAggregations().get("avg_probability"); HashMap<String, Double> map = new HashMap<>(); map.put("count", (double) docCount); map.put("sum", sum.getValue()); map.put("avg", avg.getValue()); retMap.put(key, map); } return retMap; }
public static List<String> termsKeys(Terms terms) { return terms.getBuckets() .stream() .map(Terms.Bucket::getKeyAsString) .collect(MoreCollectors.toList(terms.getBuckets().size())); }
/** * Get AggregationBuilder to calculate sum, avg and doc count for given classifier. * @param classifierName Name of classifier * @return AggregationBuilder with required configuration */ private static TermsBuilder getClassifierAggregationBuilder(String classifierName) { String probabilityField = classifierName + "_probability"; return AggregationBuilders.terms("by_class").field(classifierName) .subAggregation( AggregationBuilders.avg("avg_probability").field(probabilityField) ) .subAggregation( AggregationBuilders.sum("sum_probability").field(probabilityField) ); }
public List<String> searchTags(IssueQuery query, @Nullable String textQuery, int size) { Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, textQuery, Terms.Order.term(true), size); return EsUtils.termsKeys(terms); }