public ProduceRequestResult asyncRequest(final long baseOffset, final RuntimeException error, final long timeout) { final ProduceRequestResult request = new ProduceRequestResult(topicPartition); Thread thread = new Thread() { public void run() { try { sleep(timeout); request.set(baseOffset, RecordBatch.NO_TIMESTAMP, error); request.done(); } catch (InterruptedException e) { } } }; thread.start(); return request; }
/** * Mark all partitions as ready to send and block until the send is complete */ public void awaitFlushCompletion() throws InterruptedException { try { for (ProducerBatch batch : this.incomplete.copyAll()) batch.produceFuture.await(); } finally { this.flushesInProgress.decrementAndGet(); } }
@Override public boolean isDone() { if (nextRecordMetadata != null) return nextRecordMetadata.isDone(); return this.result.completed(); }
private long timestamp() { return result.hasLogAppendTime() ? result.logAppendTime() : createTimestamp; }
RecordMetadata value() { if (nextRecordMetadata != null) return nextRecordMetadata.value(); return new RecordMetadata(result.topicPartition(), this.result.baseOffset(), this.relativeOffset, timestamp(), this.checksum, this.serializedKeySize, this.serializedValueSize); }
/** * Test that waiting on a request that never completes times out */ @Test public void testTimeout() throws Exception { ProduceRequestResult request = new ProduceRequestResult(topicPartition); FutureRecordMetadata future = new FutureRecordMetadata(request, relOffset, RecordBatch.NO_TIMESTAMP, 0L, 0, 0, Time.SYSTEM); assertFalse("Request is not completed", future.isDone()); try { future.get(5, TimeUnit.MILLISECONDS); fail("Should have thrown exception."); } catch (TimeoutException e) { /* this is good */ } request.set(baseOffset, RecordBatch.NO_TIMESTAMP, null); request.done(); assertTrue(future.isDone()); assertEquals(baseOffset + relOffset, future.get().offset()); }
private RecordMetadata valueOrError() throws ExecutionException { if (this.result.error() != null) throw new ExecutionException(this.result.error()); else return new RecordMetadata(result.topicPartition(), this.result.baseOffset(), this.relativeOffset); }
public void complete(RuntimeException e) { result.done(topicPartition, e == null ? offset : -1L, e); if (callback != null) { if (e == null) callback.onCompletion(metadata, null); else callback.onCompletion(null, e); } } }
/** * Complete the request * * @param baseOffset The base offset of the messages assigned by the server * @param exception The exception that occurred (or null if the request was successful) */ public void done(long baseOffset, RuntimeException exception) { this.produceFuture.done(topicPartition, baseOffset, exception); log.trace("Produced messages to topic-partition {} with base offset offset {} and error: {}.", topicPartition, baseOffset, exception); // execute callbacks for (int i = 0; i < this.thunks.size(); i++) { try { Thunk thunk = this.thunks.get(i); if (exception == null) thunk.callback.onCompletion(thunk.future.get(), null); else thunk.callback.onCompletion(null, exception); } catch (Exception e) { log.error("Error executing user-provided callback on message for topic-partition {}:", topicPartition, e); } } }
private void completeFutureAndFireCallbacks(long baseOffset, long logAppendTime, RuntimeException exception) { // Set the future before invoking the callbacks as we rely on its state for the `onCompletion` call produceFuture.set(baseOffset, logAppendTime, exception); // execute callbacks for (Thunk thunk : thunks) { try { if (exception == null) { RecordMetadata metadata = thunk.future.value(); if (thunk.callback != null) thunk.callback.onCompletion(metadata, null); } else { if (thunk.callback != null) thunk.callback.onCompletion(null, exception); } } catch (Exception e) { log.error("Error executing user-provided callback on message for topic-partition '{}'", topicPartition, e); } } produceFuture.done(); }
@Override public RecordMetadata get() throws InterruptedException, ExecutionException { this.result.await(); if (nextRecordMetadata != null) return nextRecordMetadata.get(); return valueOrError(); }
public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long createdMs, boolean isSplitBatch) { this.createdMs = createdMs; this.lastAttemptMs = createdMs; this.recordsBuilder = recordsBuilder; this.topicPartition = tp; this.lastAppendTime = createdMs; this.produceFuture = new ProduceRequestResult(topicPartition); this.retry = false; this.isSplitBatch = isSplitBatch; float compressionRatioEstimation = CompressionRatioEstimator.estimation(topicPartition.topic(), recordsBuilder.compressionType()); recordsBuilder.setEstimatedCompressionRatio(compressionRatioEstimation); }
for (ProducerBatch batch : drainedEntry.getValue()) { assertTrue(batch.isClosed()); assertFalse(batch.produceFuture.completed()); numDrainedRecords += batch.recordCount;
@Override public RecordMetadata get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { // Handle overflow. long now = time.milliseconds(); long timeoutMillis = unit.toMillis(timeout); long deadline = Long.MAX_VALUE - timeoutMillis < now ? Long.MAX_VALUE : now + timeoutMillis; boolean occurred = this.result.await(timeout, unit); if (!occurred) throw new TimeoutException("Timeout after waiting for " + timeoutMillis + " ms."); if (nextRecordMetadata != null) return nextRecordMetadata.get(deadline - time.milliseconds(), TimeUnit.MILLISECONDS); return valueOrError(); }
partition = partition(record, this.cluster); TopicPartition topicPartition = new TopicPartition(record.topic(), partition); ProduceRequestResult result = new ProduceRequestResult(topicPartition); FutureRecordMetadata future = new FutureRecordMetadata(result, 0, RecordBatch.NO_TIMESTAMP, 0L, 0, 0, Time.SYSTEM);
for (ProducerBatch batch : drainedEntry.getValue()) { assertTrue(batch.isClosed()); assertFalse(batch.produceFuture.completed()); numDrainedRecords += batch.recordCount;