public static ElasticConnection getConnection(String hostname, int transportPort) { return getConnection(hostname, transportPort, DEFAULT_FLUSH_INTERVAL_STRING, DEFAULT_BULK_LISTENER); }
private static ElasticConnection getConnection(String hostname, int transportPort, String flushIntervalString, BulkProcessor.Listener listener) { System.setProperty("es.set.netty.runtime.available.processors", "false"); TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString, TimeValue.timeValueSeconds(5), "flush"); Client client = getClient(hostname, transportPort); BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener) .setFlushInterval(flushInterval) .setBulkActions(10) .setConcurrentRequests(10) .build(); return new ElasticConnection(client, bulkProcessor); }
public void close() { if (connection != null) { connection.close(); } } }
public void delete(String url) { if (url != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get(); } }
public void updateStatus(String url, String status) throws IOException { UpdateRequestBuilder update = getConnection().getClient() .prepareUpdate(getIndex(), getType(), url) .setDoc(jsonBuilder() .startObject() .field(STATUS_FIELD, status) .endObject()); getConnection().getProcessor().add(update.request()); }
public void delete(String url) { if (url != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get(); } }
public void store(HttpArticle article, Map<String, Object> fields) throws IOException { XContentBuilder jsonBuilder = jsonBuilder(); jsonBuilder.startObject(); applyFields(jsonBuilder, article, fields); jsonBuilder.endObject(); IndexRequestBuilder insert = getConnection().getClient() .prepareIndex(getIndex(), getType(), article.getUrl()) .setSource(jsonBuilder); getConnection().getProcessor().add(insert.request()); }
public PageableList<HttpSourceTest> filter(String prefix) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(prefix)) { filter.must(QueryBuilders .prefixQuery("url", prefix.trim())); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSourceTest> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
public void upsertUrlStatus(String url, String published, String source, boolean create, Enum status) throws IOException { Date now = new Date(); IndexRequestBuilder insert = getConnection().getClient() .prepareIndex(getIndex(), getType(), url) .setSource(jsonBuilder() .startObject() .field("url", url) .field("source", source) .field("created", now) .field("updated", now) .field("published", published) .field("status", String.valueOf(status)) .endObject()) .setCreate(create); UpdateRequestBuilder update = getConnection().getClient() .prepareUpdate(getIndex(), getType(), url) .setDoc(jsonBuilder() .startObject() .field("updated", now) .field("published", published) .field("status", String.valueOf(status)) .endObject()) .setUpsert(insert.request()); getConnection().getProcessor().add(create ? insert.request() : update.request()); }
public static ElasticConnection getConnection(String hostname) { return getConnection(hostname, DEFAULT_TRANSPORT_PORT, DEFAULT_FLUSH_INTERVAL_STRING, DEFAULT_BULK_LISTENER); }
public PageableList<NamedQuery> filter(String prefix) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(prefix)) { filter.must(QueryBuilders .prefixQuery("name", prefix.trim())); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<NamedQuery> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToNamedQuery) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
private static ElasticConnection getEsConnection() { if (CONNECTION == null) { String hostname = Configuration.INSTANCE.getString(ElasticConstants.ES_HOSTNAME_PARAM, "localhost"); int transportPort = Configuration.INSTANCE.getInt(ElasticConstants.ES_TRANSPORT_PORT_PARAM, 9300); CONNECTION = ElasticConnection.getConnection(hostname, transportPort); } return CONNECTION; } }
public List<String> suggest(String prefix) { CompletionSuggestionBuilder suggestionBuilder = new CompletionSuggestionBuilder("name_suggest") .prefix(prefix); SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .suggest(new SuggestBuilder().addSuggestion("suggestion", suggestionBuilder)) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); return response.getSuggest().filter(CompletionSuggestion.class).stream() .flatMap(s -> s.getOptions().stream()) .sorted(Comparator.comparingDouble(Suggest.Suggestion.Entry.Option::getScore)) .map(Suggest.Suggestion.Entry.Option::getText) .map(Text::toString) .collect(Collectors.toList()); }
public List<HttpArticle> findByStatus(String status, int count) { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(STATUS_FIELD, String.valueOf(status))); SearchResponse response = getConnection().getClient() .prepareSearch(getIndex()) .setTypes(getType()) .setSearchType(SearchType.DEFAULT) .setPostFilter(filter) .addSort(CREATED_FIELD, SortOrder.DESC) .setSize(count) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); SearchHits hits = response.getHits(); return Arrays.stream(hits.getHits()) .map(SearchHit::getSource) .map(this::mapToHttpArticle) .collect(Collectors.toList()); }
public List<NamedQuery> all() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setQuery(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<NamedQuery> result = Lists.newArrayList(); do { result.addAll(Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToNamedQuery) .collect(Collectors.toList())); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); return result; }
public PageableList<HttpSource> filter(String text) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(text)) { filter.must(QueryBuilders .queryStringQuery(QueryParser.escape(text.trim())) .field("search_field") .defaultOperator(QueryStringQueryBuilder.DEFAULT_OPERATOR.AND)); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSource> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSource) .sorted((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName())) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
public List<HttpSource> findEnabledSources() { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("enabled", true)); SearchResponse response = getConnection().getClient() .prepareSearch(getIndex()) .setTypes(getType()) .setSearchType(SearchType.DEFAULT) .setPostFilter(filter) .addSort("updated", SortOrder.ASC) .setSize(10000) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); SearchHits hits = response.getHits(); return Arrays.stream(hits.getHits()) .map(SearchHit::sourceAsMap) .map(this::mapToHttpSource) .collect(Collectors.toList()); }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSource) .map(HttpSource::getUrl) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .map(HttpSourceTest::getUrl) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
public List<HttpSourceTest> all() { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.existsQuery("source_url")); Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSourceTest> result = Lists.newArrayList(); do { result.addAll(Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .collect(Collectors.toList())); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); return result; }