private void ensureIndex(boolean createIfAbsent) throws IOException { GetIndexResponse indexResponse = getElasticClient() .admin() .indices() .prepareGetIndex() .get(elasticsearchProperties.getElasticsearchSearchTimeout()); if (Arrays.asList(indexResponse.getIndices()).stream().anyMatch(i -> i.equalsIgnoreCase(policyIndexName))) { ensureIndexAvailable(); return; } if (!createIfAbsent) { String error = MessageFormat.format("Failed to ensure that the index \"{0}\" exists in Elasticsearch", policyIndexName); logger.error(error); throw new RuntimeException(error); } logger.warn("Index \"{}\" does not exist in Elasticsearch. Creating index...", policyIndexName); try { createIndex(); } catch (IndexAlreadyExistsException e) { logger.warn("Index \"{}\" was found to exist during creation attempt.", policyIndexName); } ensureIndex(false); }
.endObject() .endObject()) .get(elasticsearchProperties.getElasticsearchSearchTimeout());
@Override public void isValidExpression(String string) { try { XContentBuilder query = BooleanExpressionParser.wrapQuery(parseQuery(string)); if (query != null) { initialize(); ValidateQueryRequest validationRequest = new ValidateQueryRequest(policyIndexName) .source(query); validationRequest.explain(true); ValidateQueryResponse validationResponse = getElasticClient() .admin() .indices() .validateQuery(validationRequest) .actionGet(elasticsearchProperties.getElasticsearchSearchTimeout()); if (!validationResponse.isValid()) { throw new RuntimeException(validationResponse .getQueryExplanation() .stream() .map(QueryExplanation::getError) .collect(Collectors.joining(System.getProperty("line.separator")))); } } } catch (IOException e) { throw new BackEndRequestFailedCpeException(e); } }
private Collection<AgentResult> getAgents(PercolateResponse response) throws UnknownHostException { Map<String, AgentResult> indexedAgents = new HashMap<>(); BoolQueryBuilder agentsQueryBuilder = QueryBuilders.boolQuery(); for (PercolateResponse.Match match : response) { String agentId = match.getId().string(); agentsQueryBuilder = agentsQueryBuilder.should(termQuery("_id", agentId)); indexedAgents.put(agentId, new AgentResult(agentId, match, null)); } SearchHits agentHits = getElasticClient() .prepareSearch(policyIndexName) .setTypes(percolatorTypeName) .setQuery(agentsQueryBuilder) .setSize(elasticsearchProperties.getElasticsearchMaxStoredqueries()) .get(elasticsearchProperties.getElasticsearchSearchTimeout()) .getHits(); for (SearchHit agentHit : agentHits) { AgentResult agentResult = indexedAgents.get(agentHit.getId()); if (agentResult != null) { agentResult.setSource(agentHit.sourceAsMap()); } } if (indexedAgents.values().stream().anyMatch(r -> r.getSource() == null)) { String error = "Failed to retrieve all the expected boolean agents from Elasticsearch"; logger.error(error); throw new RuntimeException(error); } return indexedAgents.values(); }
@Override public void delete(String instanceId) { try { String storedQueryId = getStoredQueryId(instanceId); if (storedQueryId != null) { getElasticClient() .prepareDelete(policyIndexName, percolatorTypeName, storedQueryId) .get(elasticsearchProperties.getElasticsearchSearchTimeout()); } // Note: Elasticsearch is near-realtime and there can be a default one-second delay between issuing a // deletion request and the deletion being apparent in index queries. This delay has been allowed // for in integration tests for Elasticsearch deletion rather than here in the delete implementation // itself, as the delete method may be called multiple times. } catch (IOException e) { throw new BackEndRequestFailedCpeException(e); } }
private void createStoredQuery(String instanceId, String ttl, BooleanAgentDocument booleanAgentDocument, Optional<String> booleanRestriction) throws IOException { String reference = booleanAgentDocument.getReference() + "_" + instanceId; XContentBuilder storedQuery = prepareStoredQuery(instanceId, booleanAgentDocument, booleanRestriction, reference); // replaces an existing document with a new document by using the reference value as the Elasticsearch _id. getElasticClient() .prepareIndex(policyIndexName, percolatorTypeName, reference) .setSource(storedQuery) .setRefresh(true) .setTTL(ttl) .get(elasticsearchProperties.getElasticsearchSearchTimeout()); }
private TermVectorsResponse getTermVectors(String text) throws IOException { return getElasticClient() .prepareTermVectors() .setIndex(policyIndexName) .setType(percolatorTypeName) .setDfs(true) .setTermStatistics(true) .setFieldStatistics(false) .setPositions(true) .setOffsets(true) .setPayloads(false) .setDoc(jsonBuilder() .startObject() .field(booleanRestrictionFieldName, text) .endObject()) .get(elasticsearchProperties.getElasticsearchSearchTimeout()); }
private String getStoredQueryId(String instanceId) { try { initialize(); QueryBuilder agentQueryBuilder = matchQuery(instanceIdFieldName, instanceId); SearchHits agentHits = getElasticClient() .prepareSearch(policyIndexName) .setTypes(percolatorTypeName) .setQuery(agentQueryBuilder) .setSize(1) .get(elasticsearchProperties.getElasticsearchSearchTimeout()) .getHits(); if (agentHits.getTotalHits() == 0) { return null; } return agentHits.getAt(0).getId(); } catch (IOException e) { throw new BackEndRequestFailedCpeException(e); } }
private void queryBooleanAgents(String instanceId, String textToQuery, BooleanAgentQueryResult booleanAgentQueryResult) throws IOException { PercolateResponse response = getElasticClient() .preparePercolate() .setIndices(policyIndexName) .setDocumentType(dbTypeName) .setPercolateDoc(PercolateSourceBuilder.docBuilder() .setDoc(XContentFactory.jsonBuilder() .startObject() .field(contentFieldName, textToQuery) .endObject())) .setPercolateQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.boolQuery() .must(matchQuery(projectIdFieldName, userContext.getProjectId() + "_" + instanceId)) .must(matchQuery(instanceIdFieldName, instanceId)))) .setHighlightBuilder(new HighlightBuilder() .field(contentFieldName) .preTags(getStartTagGuid()) .postTags(getEndTagGuid())) .setSize(elasticsearchProperties.getElasticsearchMaxStoredqueryResults()) .get(elasticsearchProperties.getElasticsearchSearchTimeout()); List<BooleanAgentDocument> booleanAgentDocuments = convertToBooleanAgentDocuments(response); booleanAgentDocuments.forEach(d -> extractTermsFromBooleanAgentDocument(textToQuery, booleanAgentQueryResult, d)); }