/** * Delete documents using a query. Check what would be deleted first with a normal search query! * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed * in later versions. Instead, there is a plugin which iterates over search results, * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html * We simulate the same behaviour here without the need of that plugin. * * @param q * @return delete document count */ public int deleteByQuery(String indexName, final QueryBuilder q) { Map<String, String> ids = new TreeMap<>(); // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`" @SuppressWarnings("deprecation") SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet(); while (true) { // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the // scroll for (SearchHit hit : response.getHits().getHits()) { ids.put(hit.getId(), hit.getType()); } response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet(); // termination if (response.getHits().getHits().length == 0) break; } return deleteBulk(indexName, ids); }
.setBulkActions(10000) .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) .setFlushInterval(TimeValue.timeValueSeconds(5)) .setConcurrentRequests(10) .setBackoffPolicy( BackoffPolicy.exponentialBackoff( TimeValue.timeValueMillis(100), 3)) .build();
@Override public <T> CloseableIterator<T> stream(CriteriaQuery query, Class<T> clazz) { final long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis(); return doStream(scrollTimeInMillis, (ScrolledPage<T>) startScroll(scrollTimeInMillis, query, clazz), clazz, resultsMapper); }
LOGGER.info("Check if "+searchguardIndex+" index exists ..."); .masterNodeTimeout(TimeValue.timeValueMinutes(1)); threadContext.putHeader(ConfigConstants.SG_CONF_REQUEST_HEADER, "true"); client.admin().indices().exists(ier, new ActionListener<IndicesExistsResponse>() { LOGGER.error("Failure while executing IndicesExistsRequest {}",e2, e2); bgThread.start();
final int oldCapacity = workQueue.capacity(); if (logger.isDebugEnabled()) { final long avgTaskTime = totalNanos / tasksPerFrame; logger.debug("[{}]: there were [{}] tasks in [{}], avg task time [{}], EWMA task execution [{}], " + "[{} tasks/s], optimal queue is [{}], current capacity [{}]", getName(), tasksPerFrame, TimeValue.timeValueNanos(totalRuntime), TimeValue.timeValueNanos(avgTaskTime), TimeValue.timeValueNanos((long)executionEWMA.getAverage()), String.format(Locale.ROOT, "%.2f", lambda * TimeValue.timeValueSeconds(1).nanos()), desiredQueueSize, oldCapacity); if (oldCapacity != newCapacity && logger.isDebugEnabled()) { logger.debug("adjusted [{}] queue size by [{}], old capacity: [{}], new capacity: [{}]", getName(), newCapacity > oldCapacity ? QUEUE_ADJUSTMENT_AMOUNT : -QUEUE_ADJUSTMENT_AMOUNT,
public List<String> findAllViewUuids() { SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW) .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .setFetchSource(false) .setSize(100) .setQuery(matchAllQuery()); SearchResponse response = esSearch.get(); List<String> result = newArrayList(); while (true) { List<SearchHit> hits = newArrayList(response.getHits()); for (SearchHit hit : hits) { result.add(hit.getId()); } String scrollId = response.getScrollId(); response = esClient.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .get(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break; } } return result; } }
public QueryIterator(FilterBuilder filter, int startFrom, int scrollSize, long maxSize, Client client, Function<Iterator<SearchHit>, Iterator<? extends E>> convertFunc, Boolean refresh, TimingAccessor timing, String... indices) { this.client = client; this.allowedRemaining = maxSize; this.convertFunc = convertFunc; this.timing = timing; if (refresh) client.admin().indices().prepareRefresh(indices).execute().actionGet(); this.timing.start("scroll"); scrollResponse = client.prepareSearch(indices) .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)) .setFrom(startFrom) .setScroll(new TimeValue(60000)) .setSize(maxSize < scrollSize ? (int) maxSize : scrollSize) .execute().actionGet(); this.timing.stop("scroll"); hits = convertFunc.apply(scrollResponse.getHits().iterator()); }
@Override public SearchResponse search(QueryBuilder queryBuilder, Integer limit) { TimeValue timeout = TimeValue.timeValueSeconds(7); return client.prepareSearch("photon"). setSearchType(SearchType.QUERY_AND_FETCH). setQuery(queryBuilder). setSize(limit). setTimeout(timeout). execute(). actionGet(); }
@Override public boolean hasNext() { if(allowedRemaining <= 0) return false; if(hits.hasNext()) return true; timing.start("scroll"); scrollResponse = client.prepareSearchScroll(scrollResponse.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); timing.stop("scroll"); hits = convertFunc.apply(scrollResponse.getHits().iterator()); return hits.hasNext(); }
private List<SearchHit> scrollTillLimit(TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { SearchResponse scrollResp = scrollOneTimeWithMax(client,tableInJoinRequest); updateMetaSearchResults(scrollResp); List<SearchHit> hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); if (hintLimit == null) hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; while (hits.length != 0 && curentNumOfResults < hintLimit) { curentNumOfResults += hits.length; Collections.addAll(hitsWithScan, hits); if (curentNumOfResults >= MAX_RESULTS_FOR_FIRST_TABLE) { //todo: log or exception? System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits(); } return hitsWithScan; }
Assert.assertEquals(1, actionGet.getHits().getHits().length); System.out.println("------- 6 ---------"); Assert.assertEquals(0, actionGet.getHits().getHits().length); try { tc.threadPool().getThreadContext().putHeader("sg_impersonate_as", "nagilum"); SearchResponse searchRes = tc.prepareSearch("starfleet").setTypes("ships").setScroll(TimeValue.timeValueMinutes(5)).get(); scrollId = searchRes.getScrollId(); } finally { ctx.close(); try { tc.threadPool().getThreadContext().putHeader("sg_impersonate_as", "nagilum"); searchRes = tc.prepareSearch("starfleet").setTypes("ships").setScroll(TimeValue.timeValueMinutes(5)).get(); } finally { ctx.close(); try { tc.threadPool().getThreadContext().putHeader("sg_impersonate_as", "nagilum"); searchRes = tc.prepareSearch("starfleet").setTypes("ships").setScroll(TimeValue.timeValueMinutes(5)).get(); SearchResponse scrollRes = tc.prepareSearchScroll(searchRes.getScrollId()).get(); Assert.assertEquals(0, scrollRes.getFailedShards()); } finally {
boolean finishedScrolling; if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { searchResponse = secondTableRequest.getRequestBuilder().setSize(hintLimit).get(); finishedScrolling = true; } else { searchResponse = secondTableRequest.getRequestBuilder() .setScroll(new TimeValue(60000)) .setSize(MAX_RESULTS_ON_ONE_FETCH).get(); int fetchedSoFarFromSecondTable = 0; while (!limitReached) { SearchHit[] secondTableHits = searchResponse.getHits().getHits(); fetchedSoFarFromSecondTable += secondTableHits.length; for (SearchHit secondTableHit : secondTableHits) { searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); } else break; } else {
public ProcessService(Settings settings) { this.probe = ProcessProbe.getInstance(); final TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings); processStatsCache = new ProcessStatsCache(refreshInterval, probe.processStats()); this.info = probe.processInfo(refreshInterval.millis()); logger.debug("using refresh_interval [{}]", refreshInterval); }
protected <E extends EsDocument> ScrollableResponse<List<E>> retrieveAll( String[] includeFields, int size, ThrowingFunction<String, E> createFunc) throws Exception { Preconditions.checkArgument(size > 0); SearchRequestBuilder builder = esClient.prepareSearch() .setIndices(getIndexName()).setTypes(getDocTypeName()) .setScroll(new TimeValue(SCROLLDEFAULTTIMEOUT)) .setSize(size) .setFetchSource(includeFields, null).setVersion(true); SearchResponse response = builder.execute().actionGet(); return convertToScrollableResponse(response, createFunc); }
TransportClientNodesService(Settings settings, TransportService transportService, ThreadPool threadPool, TransportClient.HostFailureListener hostFailureListener) { this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings); this.transportService = transportService; this.threadPool = threadPool; this.minCompatibilityVersion = Version.CURRENT.minimumCompatibilityVersion(); this.nodesSamplerInterval = TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(settings); this.pingTimeout = TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.get(settings).millis(); this.ignoreClusterName = TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(settings); if (logger.isDebugEnabled()) { logger.debug("node_sampler_interval[{}]", nodesSamplerInterval); } if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(settings)) { this.nodesSampler = new SniffNodesSampler(); } else { this.nodesSampler = new SimpleNodeSampler(); } this.hostFailureListener = hostFailureListener; this.nodesSamplerFuture = threadPool.schedule(nodesSamplerInterval, ThreadPool.Names.GENERIC, new ScheduledNodeSampler()); }
public SearchRequestBuilder buildScrollSearchRequest() { String indices = index != null && !index.isEmpty() ? index : "_all"; List<String> fields = columns.stream() .map(ElasticsearchColumnHandle::getColumnName) .collect(toList()); SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indices) .setTypes(type) .setSearchType(QUERY_THEN_FETCH) .setScroll(new TimeValue(scrollTimeout.toMillis())) .setFetchSource(fields.toArray(new String[0]), null) .setQuery(buildSearchQuery()) .setPreference("_shards:" + shard) .setSize(scrollSize); LOG.debug("Elasticsearch Request: %s", searchRequestBuilder); return searchRequestBuilder; }
@Override public SearchResponse executeESScrollSearchNextRequest(SearchResponse scrollResp) { return client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(ES_SCROLL_KEEPALIVE)).execute() .actionGet(); }
@SuppressWarnings("PMD.NcssMethodCount") SearchResponse startScroll() { //https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName); searchRequestBuilder.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) .setQuery(createQuery()) .setSize(BATCH_SIZE) .setExplain(false) .setFetchSource(false) .setVersion(true) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); return searchRequestBuilder.execute().actionGet(); }
final int halfProcMaxAt10 = halfNumberOfProcessorsMaxTen(availableProcessors); final int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512); builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200, true)); builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, "bulk", availableProcessors, 200)); builders.put(Names.SEARCH_THROTTLED, new AutoQueueAdjustingExecutorBuilder(settings, Names.SEARCH_THROTTLED, 1, 100, 100, 100, 200)); builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5))); builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5))); builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); builders.put(Names.FETCH_SHARD_STARTED, new ScalingExecutorBuilder(Names.FETCH_SHARD_STARTED, 1, 2 * availableProcessors, TimeValue.timeValueMinutes(5))); builders.put(Names.FORCE_MERGE, new FixedExecutorBuilder(settings, Names.FORCE_MERGE, 1, -1)); builders.put(Names.FETCH_SHARD_STORE, new ScalingExecutorBuilder(Names.FETCH_SHARD_STORE, 1, 2 * availableProcessors, TimeValue.timeValueMinutes(5))); for (final ExecutorBuilder<?> builder : customBuilders) { if (builders.containsKey(builder.name())) { throw new IllegalStateException("duplicate executors with name [" + executorHolder.info.getName() + "] registered"); this.scheduler = Scheduler.initScheduler(settings); TimeValue estimatedTimeInterval = ESTIMATED_TIME_INTERVAL_SETTING.get(settings); this.cachedTimeThread = new CachedTimeThread(EsExecutors.threadName(settings, "[timer]"), estimatedTimeInterval.millis()); this.cachedTimeThread.start();