final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable(); ShardRouting targetShardRouting = routingTable.getByAllocationId(request.targetAllocationId()); if (targetShardRouting == null) { logger.debug("delaying recovery of {} as it is not listed as assigned to target node {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); try (Closeable ignored = shard.acquireRetentionLockForPeerRecovery()) { final long startingSeqNo; final long requiredSeqNoRangeStart; final boolean isSequenceNumberBasedRecovery = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && isTargetSameHistory() && shard.hasCompleteHistoryOperations("peer-recovery", request.startingSeqNo()); if (isSequenceNumberBasedRecovery) { logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo()); startingSeqNo = request.startingSeqNo(); requiredSeqNoRangeStart = startingSeqNo; final Engine.IndexCommitRef phase1Snapshot; try { phase1Snapshot = shard.acquireSafeIndexCommit(); } catch (final Exception e) { throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e); startingSeqNo = shard.indexSettings().isSoftDeleteEnabled() ? requiredSeqNoRangeStart : 0; try { final int estimateNumOps = shard.estimateNumberOfHistoryOperations("peer-recovery", startingSeqNo); phase1(phase1Snapshot.getIndexCommit(), () -> estimateNumOps); } catch (final Exception e) { throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
public Engine.Searcher acquireSearcher(String source) { return acquireSearcher(source, Engine.SearcherScope.EXTERNAL); }
private void maybeSyncGlobalCheckpoints() { for (final IndexShard shard : this.shards.values()) { if (shard.routingEntry().active() && shard.routingEntry().primary()) { switch (shard.state()) { case CLOSED: case CREATED: continue; case POST_RECOVERY: assert false : "shard " + shard.shardId() + " is in post-recovery but marked as active"; continue; case STARTED: try { shard.acquirePrimaryOperationPermit( ActionListener.wrap( releasable -> { try (Releasable ignored = releasable) { shard.maybeSyncGlobalCheckpoint("background"); logger.info( new ParameterizedMessage( "{} failed to execute background global checkpoint sync", shard.shardId()), e); throw new IllegalStateException("unknown state [" + shard.state() + "]");
/** * ask this shard to check now whether it is inactive, and reduces its indexing buffer if so. */ protected void checkIdle(IndexShard shard, long inactiveTimeNS) { try { shard.checkIdle(inactiveTimeNS); } catch (AlreadyClosedException e) { logger.trace(() -> new ParameterizedMessage("ignore exception while checking if shard {} is inactive", shard.shardId()), e); } } }
private boolean canRecover(IndexShard indexShard) { if (indexShard.state() == IndexShardState.CLOSED) { // got closed on us, just ignore this recovery return false; } if (indexShard.routingEntry().primary() == false) { throw new IndexShardRecoveryException(shardId, "Trying to recover when the shard is in backup state", null); } return true; }
private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) { final int shardId = sId.id(); final Settings indexSettings = this.getIndexSettings().getSettings(); try { try { indexShard.close(reason, flushEngine); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("[{}] failed to close index shard", shardId), e); store.close(); } else { logger.trace("[{}] store not initialized prior to closing shard, nothing to close", shardId); logger.warn( () -> new ParameterizedMessage( "[{}] failed to close store on shard removal (reason: [{}])", shardId, reason), e);
@Override public void onShardInactive(IndexShard indexShard) { for (IndexEventListener listener : listeners) { try { listener.onShardInactive(indexShard); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke on shard inactive callback", indexShard.shardId().getId()), e); throw e; } } }
void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) { if (shard.state() == IndexShardState.CLOSED) { return; if (settings.isWarmerEnabled() == false) { return; if (logger.isTraceEnabled()) { logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader()); shard.warmerService().onPreWarm(); long time = System.nanoTime(); final List<TerminationHandle> terminationHandles = new ArrayList<>(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.warn("top warming has been interrupted", e); break; shard.warmerService().onPostWarm(took); if (shard.warmerService().logger().isTraceEnabled()) { shard.warmerService().logger().trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
@Override public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) { if (indexSettings.getIndex().equals(indexShard.indexSettings().getIndex()) == false) { final MapperService mapperService = indexShard.mapperService(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { if (docMapper.hasNestedObjects()) { warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated())); final long start = System.nanoTime(); getAndLoadIfNotPresent(filterToWarm, ctx); if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace("warmed bitset for [{}], took [{}]", filterToWarm, TimeValue.timeValueNanos(System.nanoTime() - start)); indexShard.warmerService().logger().warn(() -> new ParameterizedMessage("failed to load " + "bitset for [{}]", filterToWarm), e); } finally {
@Override public TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) { final MapperService mapperService = indexShard.mapperService(); final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>(); if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace( "warmed global ordinals for [{}], took [{}]", fieldType.name(), .warmerService() .logger() .warn(() -> new ParameterizedMessage("failed to warm-up global ordinals for [{}]", fieldType.name()), e); } finally { latch.countDown();
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary"); } if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); } RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard); logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode()); try { return handler.recoverToTarget(); } finally { ongoingRecoveries.remove(shard, handler); } }
private PreSyncedFlushResponse performPreSyncedFlush(PreShardSyncedFlushRequest request) { IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id()); FlushRequest flushRequest = new FlushRequest().force(false).waitIfOngoing(true); logger.trace("{} performing pre sync flush", request.shardId()); indexShard.flush(flushRequest); final CommitStats commitStats = indexShard.commitStats(); final Engine.CommitId commitId = commitStats.getRawCommitId(); logger.trace("{} pre sync flush done. commit id {}, num docs {}", request.shardId(), commitId, commitStats.getNumDocs()); return new PreSyncedFlushResponse(commitId, commitStats.getNumDocs(), commitStats.syncId()); }
@Override public void onFailure(Exception e) { logger.debug(() -> new ParameterizedMessage("{} sync flush on inactive shard failed", indexShard.shardId()), e); } });
@Override public void onFailure(Exception e) { logger.warn(() -> new ParameterizedMessage("failed to write indexing buffer for shard [{}]; ignoring", shard.shardId()), e); } });
boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, final IndexShard indexShard, final List<LocalShardSnapshot> shards) throws IOException { if (canRecover(indexShard)) { RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType(); assert recoveryType == RecoverySource.Type.LOCAL_SHARDS: "expected local shards recovery type: " + recoveryType; if (shards.isEmpty()) { indexShard.mapperService().merge(sourceMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); Sort indexSort = indexShard.getIndexSort(); final boolean hasNested = indexShard.mapperService().hasNested(); final boolean isSplit = sourceMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); assert isSplit == false || sourceMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + "single type but the index is created before 6.0.0"; return executeRecovery(indexShard, () -> { logger.debug("starting recovery from local shards {}", shards); try { final Directory directory = indexShard.store().directory(); // don't close this directory!! final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new); final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong(); final long maxUnsafeAutoIdTimestamp = shards.stream().mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp).max().getAsLong(); addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp, indexShard.indexSettings().getIndexMetaData(), indexShard.shardId().id(), isSplit, hasNested); internalRecoverFromStore(indexShard); indexShard.getEngine().forceMerge(false, -1, false, false, false); } catch (IOException ex) { throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
@Override protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (request.refresh() && !request.realtime()) { indexShard.refresh("refresh_flag_mget"); } MultiGetShardResponse response = new MultiGetShardResponse(); for (int i = 0; i < request.locations.size(); i++) { MultiGetRequest.Item item = request.items.get(i); try { GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(), item.version(), item.versionType(), item.fetchSourceContext()); response.add(request.locations.get(i), new GetResponse(getResult)); } catch (RuntimeException e) { if (TransportActions.isShardNotAvailableException(e)) { throw e; } else { logger.debug(() -> new ParameterizedMessage("{} failed to execute multi_get for [{}]/[{}]", shardId, item.type(), item.id()), e); response.add(request.locations.get(i), new MultiGetResponse.Failure(request.index(), item.type(), item.id(), e)); } } } return response; }
@Override public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) { final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability(); final boolean updateIndexMetaData = indexSettings.updateIndexMetaData(newIndexMetaData); for (final IndexShard shard : this.shards.values()) { try { shard.onSettingsChanged(); } catch (Exception e) { logger.warn( () -> new ParameterizedMessage( "[{}] failed to notify shard about setting change", shard.shardId().id()), e); if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) { rescheduleRefreshTasks();
private InFlightOpsResponse performInFlightOps(InFlightOpsRequest request) { IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().id()); if (indexShard.routingEntry().primary() == false) { throw new IllegalStateException("[" + request.shardId() +"] expected a primary shard"); } int opCount = indexShard.getActiveOperationsCount(); return new InFlightOpsResponse(opCount); }