private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) { try { logger.warn(() -> new ParameterizedMessage( "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); failedShardsCache.put(shardRouting.shardId(), shardRouting); shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state); } catch (Exception inner) { if (failure != null) inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to mark shard as failed (because of [{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } }
totalShards += nodes.get(exception.nodeId()).size(); for (ShardRouting shard : nodes.get(exception.nodeId())) { exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure, ClusterState state) { try { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex()); if (indexService != null) { indexService.removeShard(shardRouting.shardId().id(), message); } } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Exception inner) { inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to remove shard after failure ([{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } if (sendShardFailure) { sendFailShard(shardRouting, message, failure, state); } }
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { builder.field("index", shardRouting.getIndexName()); builder.field("shard", shardRouting.getId()); builder.field("primary", shardRouting.primary()); builder.field("current_state", shardRouting.state().toString().toLowerCase(Locale.ROOT));
table.addCell(shardSegment.getShardRouting().getId()); table.addCell(shardSegment.getShardRouting().primary() ? "p" : "r"); table.addCell(nodes.get(shardSegment.getShardRouting().currentNodeId()).getHostAddress());
private void failAndRemoveShard(ShardRouting shardRouting, IndexService indexService, boolean sendShardFailure, String message, @Nullable Throwable failure) { if (indexService.hasShard(shardRouting.getId())) { try { indexService.removeShard(shardRouting.getId(), message); } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Throwable e1) { logger.warn("[{}][{}] failed to remove shard after failure ([{}])", e1, shardRouting.getIndex(), shardRouting.getId(), message); } } if (sendShardFailure) { sendFailShard(shardRouting, indexService.indexUUID(), message, failure); } }
private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) { try { logger.warn(() -> new ParameterizedMessage( "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); failedShardsCache.put(shardRouting.shardId(), shardRouting); shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state); } catch (Exception inner) { if (failure != null) inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to mark shard as failed (because of [{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } }
private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) { try { logger.warn(() -> new ParameterizedMessage( "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); failedShardsCache.put(shardRouting.shardId(), shardRouting); shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state); } catch (Exception inner) { if (failure != null) inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to mark shard as failed (because of [{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } }
@Override public int compare(ShardRouting o1, ShardRouting o2) { if (o1.primary() ^ o2.primary()) { return o1.primary() ? -1 : o2.primary() ? 1 : 0; } final int indexCmp; if ((indexCmp = o1.index().compareTo(o2.index())) == 0) { return o1.getId() - o2.getId(); } // this comparator is more expensive than all the others up there // that's why it's added last even though it could be easier to read // if we'd apply it earlier. this comparator will only differentiate across // indices all shards of the same index is treated equally. final int secondary = secondaryComparator.compare(o1, o2); return secondary == 0 ? indexCmp : secondary; } };
totalShards += nodes.get(exception.nodeId()).size(); for (ShardRouting shard : nodes.get(exception.nodeId())) { exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
totalShards += nodes.get(exception.nodeId()).size(); for (ShardRouting shard : nodes.get(exception.nodeId())) { exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
totalShards += nodes.get(exception.nodeId()).size(); for (ShardRouting shard : nodes.get(exception.nodeId())) { exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
private void sendFailShard(ShardRouting shardRouting, String indexUUID, String message, @Nullable Throwable failure) { try { logger.warn("[{}] marking and sending shard failed due to [{}]", failure, shardRouting.shardId(), message); failedShards.put(shardRouting.shardId(), new FailedShard(shardRouting.version())); shardStateAction.shardFailed(shardRouting, indexUUID, message, failure); } catch (Throwable e1) { logger.warn("[{}][{}] failed to mark shard as failed (because of [{}])", e1, shardRouting.getIndex(), shardRouting.getId(), message); } }
private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure, ClusterState state) { try { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex()); if (indexService != null) { indexService.removeShard(shardRouting.shardId().id(), message); } } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Exception inner) { inner.addSuppressed(failure); logger.warn( (Supplier<?>) () -> new ParameterizedMessage( "[{}][{}] failed to remove shard after failure ([{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } }
private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure, ClusterState state) { try { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex()); if (indexService != null) { indexService.removeShard(shardRouting.shardId().id(), message); } } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Exception inner) { inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to remove shard after failure ([{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } if (sendShardFailure) { sendFailShard(shardRouting, message, failure, state); } }
private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure, ClusterState state) { try { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex()); if (indexService != null) { indexService.removeShard(shardRouting.shardId().id(), message); } } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Exception inner) { inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to remove shard after failure ([{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } if (sendShardFailure) { sendFailShard(shardRouting, message, failure, state); } }
@Test public void testMultipleShardSearch() throws Exception { Random random = new Random(); int docs = 3 + random.nextInt(10001 - 3);; int batchSize = 1 + random.nextInt(5000 - 1); int shards = 1 + random.nextInt(5 - 1); long totalHitsReceived = 0; elastic.populate(shards, 0, schema, table, docs, ElasticsearchCluster.PRIMITIVE_TYPES); assertThat(elastic.search(schema, table).count, equalTo((long) docs)); ClusterState state = state(schema); IndexRoutingTable irt = routing(state, schema); ShardsIterator iter = irt.randomAllActiveShardsIt(); DiscoveryNodes nodes = state.nodes(); assertThat(iter.size(), equalTo(shards)); ShardRouting routing; while ((routing = iter.nextOrNull()) != null) { DiscoveryNode dn = nodes.get(routing.currentNodeId()); logger.info("--> executing search on: [{}] batch size: [{}]", routing, batchSize); Search<byte[]> search = new SearchBytes() .setQuery(String.format("{ \"query\": %s } ", QueryBuilders.matchAllQuery().toString())) .setResource(schema + "/" + table) .setParameter("scroll", "10s") .setParameter("preference", "_shards:" + routing.getId()) .setParameter("size", Integer.toString(batchSize)); byte[] result = connection.execute(search); totalHitsReceived += ElasticsearchCluster.asJsonObject(result).get("hits").getAsJsonObject().get("total").getAsInt(); } logger.info("--> total hits received across [{}] shards: [{}]", iter.size(), totalHitsReceived); assertThat(totalHitsReceived, equalTo((long) docs)); }
table.addCell(shardSegment.getShardRouting().getId()); table.addCell(shardSegment.getShardRouting().primary() ? "p" : "r"); table.addCell(nodes.get(shardSegment.getShardRouting().currentNodeId()).getHostAddress());
table.addCell(shardSegment.getShardRouting().getId()); table.addCell(shardSegment.getShardRouting().primary() ? "p" : "r"); table.addCell(nodes.get(shardSegment.getShardRouting().currentNodeId()).getHostAddress());