private CreateVisitorMessage createMessage(VisitorIterator.BucketProgress bucket) { CreateVisitorMessage msg = new CreateVisitorMessage( params.getVisitorLibrary(), getNextVisitorId(), receiver.getConnectionSpec(), dataDestination); msg.getTrace().setLevel(params.getTraceLevel()); msg.setTimeRemaining(messageTimeoutMs); msg.setBuckets(Arrays.asList(bucket.getSuperbucket(), bucket.getProgress())); msg.setDocumentSelection(params.getDocumentSelection()); msg.setBucketSpace(params.getBucketSpace()); msg.setFromTimestamp(params.getFromTimestamp()); msg.setToTimestamp(params.getToTimestamp()); msg.setMaxPendingReplyCount(params.getMaxPending()); msg.setFieldSet(params.fieldSet()); msg.setVisitInconsistentBuckets(params.visitInconsistentBuckets()); msg.setVisitRemoves(params.visitRemoves()); msg.setParameters(params.getLibraryParameters()); msg.setRoute(params.getRoute()); msg.setVisitorOrdering(params.getVisitorOrdering()); msg.setMaxBucketsPerVisitor(params.getMaxBucketsPerVisitor()); msg.setLoadType(params.getLoadType()); msg.setPriority(params.getPriority()); msg.setRetryEnabled(false); return msg; }
@Override protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { CreateVisitorMessage msg = (CreateVisitorMessage)obj; encodeString(msg.getLibraryName(), buf); encodeString(msg.getInstanceId(), buf); encodeString(msg.getControlDestination(), buf); encodeString(msg.getDataDestination(), buf); encodeString(msg.getDocumentSelection(), buf); buf.putInt(null, msg.getMaxPendingReplyCount()); buf.putInt(null, msg.getBuckets().size()); for (BucketId id : msg.getBuckets()) { long rawid = id.getRawId(); long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | buf.putLong(null, msg.getFromTimestamp()); buf.putLong(null, msg.getToTimestamp()); buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); encodeString(msg.getFieldSet(), buf); buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); buf.putInt(null, msg.getParameters().size()); for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { encodeString(pairs.getKey(), buf); byte[] b = pairs.getValue(); buf.putInt(null, msg.getVisitorOrdering()); buf.putInt(null, msg.getMaxBucketsPerVisitor()); return encodeBucketSpace(msg.getBucketSpace(), buf);
@Override protected DocumentMessage doDecode(DocumentDeserializer buf) { CreateVisitorMessage msg = new CreateVisitorMessage(); msg.setLibraryName(decodeString(buf)); msg.setInstanceId(decodeString(buf)); msg.setControlDestination(decodeString(buf)); msg.setDataDestination(decodeString(buf)); msg.setDocumentSelection(decodeString(buf)); msg.setMaxPendingReplyCount(buf.getInt(null)); ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); msg.getBuckets().add(new BucketId(rawid)); msg.setFromTimestamp(buf.getLong(null)); msg.setToTimestamp(buf.getLong(null)); msg.setVisitRemoves(buf.getByte(null) == (byte)1); msg.setFieldSet(decodeString(buf)); msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); String key = decodeString(buf); int sz = buf.getInt(null); msg.getParameters().put(key, buf.getBytes(null, sz)); msg.setVisitorOrdering(buf.getInt(null)); msg.setMaxBucketsPerVisitor(buf.getInt(null)); msg.setVisitorDispatcherVersion(50); msg.setBucketSpace(decodeBucketSpace(buf)); return msg;
private void handleErrorReply(Reply reply) { CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); // Must reset bucket progress back to what it was before sending. BucketId bucket = msg.getBuckets().get(0); BucketId subProgress = msg.getBuckets().get(1); progress.getIterator().update(bucket, subProgress); String message = getErrorMessage(reply.getError(0)); log.log(LogLevel.DEBUG, sessionName + ": received error reply for bucket " + bucket + " with message '" + message + "'"); if (isFatalError(reply)) { if (params.skipBucketsOnFatalErrors()) { markBucketProgressAsFailed(bucket, subProgress, message); } else { reportVisitorError(message); transitionTo(new StateDescription(State.FAILED, message)); return; // no additional visitors will be scheduled post-failure } } if (isErrorOfType(reply, DocumentProtocol.ERROR_WRONG_DISTRIBUTION)) { handleWrongDistributionReply((WrongDistributionReply)reply); } else { if (shouldReportError(reply)) { reportVisitorError(message); } // Wait 100ms before new visitor task is executed. Will prevent // visitors from being scheduled from caller. scheduleSendCreateVisitorsIfApplicable(100, TimeUnit.MILLISECONDS); } }
@SuppressWarnings("deprecation") private BucketId getBucketId(Message msg) { switch (msg.getType()) { case DocumentProtocol.MESSAGE_PUTDOCUMENT: return factory.getBucketId(((PutDocumentMessage)msg).getDocumentPut().getDocument().getId()); case DocumentProtocol.MESSAGE_GETDOCUMENT: return factory.getBucketId(((GetDocumentMessage)msg).getDocumentId()); case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: return factory.getBucketId(((RemoveDocumentMessage)msg).getDocumentId()); case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: return factory.getBucketId(((UpdateDocumentMessage)msg).getDocumentUpdate().getId()); case DocumentProtocol.MESSAGE_GETBUCKETLIST: return ((GetBucketListMessage)msg).getBucketId(); case DocumentProtocol.MESSAGE_STATBUCKET: return ((StatBucketMessage)msg).getBucketId(); case DocumentProtocol.MESSAGE_CREATEVISITOR: return ((CreateVisitorMessage)msg).getBuckets().get(0); case DocumentProtocol.MESSAGE_REMOVELOCATION: return ((RemoveLocationMessage)msg).getBucketId(); default: log.log(LogLevel.ERROR, "Message type '" + msg.getType() + "' not supported."); return null; } }
private void handleCreateVisitorReply(CreateVisitorReply reply) { CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); BucketId superbucket = msg.getBuckets().get(0); BucketId subBucketProgress = reply.getLastBucket(); log.log(LogLevel.DEBUG, sessionName + ": received CreateVisitorReply for bucket " + superbucket + " with progress " + subBucketProgress); progress.getIterator().update(superbucket, subBucketProgress); params.getControlHandler().onProgress(progress.getToken()); statistics.add(reply.getVisitorStatistics()); params.getControlHandler().onVisitorStatistics(statistics); trace.getRoot().addChild(reply.getTrace().getRoot()); if (params.getDynamicallyIncreaseMaxBucketsPerVisitor() && (reply.getVisitorStatistics().getDocumentsReturned() < params.getMaxFirstPassHits() / 2.0)) { // Attempt to increase parallelism to reduce latency of visiting // Ensure new count is within [1, 128] int newMaxBuckets = Math.max(Math.min((int)(params.getMaxBucketsPerVisitor() * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1); params.setMaxBucketsPerVisitor(newMaxBuckets); log.log(LogLevel.DEBUG, sessionName + ": increasing max buckets per visitor to " + params.getMaxBucketsPerVisitor()); } }