@Override public void close() { client.close(); } }
@Override public Future<DLSN> apply(VALUE request) { return client.write(stream, ByteBuffer.wrap(request.toString().getBytes(UTF_8))); } }
synchronized String sendWriteToStream(int streamId) { String stream = getStream(streamId); client.writeRecordSet(stream, recordSet) .addEventListener(this); return stream; }
@Override public void run() { LOG.info("Started writer {}.", idx); while (running) { rateLimiter.getLimiter().acquire(batchSize); String streamName = streamNames.get(random.nextInt(numStreams)); final long requestMillis = System.currentTimeMillis(); final List<ByteBuffer> data = buildBufferList(batchSize, requestMillis, messageSizeBytes); if (null == data) { break; } List<Future<DLSN>> results = dlc.writeBulk(streamName, data); for (Future<DLSN> result : results) { result.addEventListener(new TimedRequestHandler(streamName, requestMillis)); } } dlc.close(); } }
@Override public void run() { LOG.info("Started writer {}.", idx); while (running) { this.limiter.getLimiter().acquire(); final String streamName = streamNames.get(random.nextInt(numStreams)); final long requestMillis = System.currentTimeMillis(); final ByteBuffer data = buildBuffer(requestMillis, messageSizeBytes); if (null == data) { break; } if (null != writer) { writer.write(data).addEventListener( new TimedRequestHandler(streamName, requestMillis)); } else { dlc.write(streamName, data).addEventListener( new TimedRequestHandler(streamName, requestMillis)); } } if (null != writer) { writer.close(); } dlc.close(); } }
client.writeRecordSet(streamName, recordSetWriter) .addEventListener(new FutureEventListener<DLSN>() { @Override ); FutureUtils.result(Future.collect(writeFutures)); client.close();
@Override protected int runCmd(DistributedLogClient client) throws Exception { System.out.println("Truncating streams : " + streams); for (String stream : streams) { boolean success = Await.result(client.truncate(stream, dlsn)); System.out.println("Truncate " + stream + " to " + dlsn + " : " + success); } return 0; }
@Override protected int runCmd(DistributedLogClient client) throws Exception { RateLimiter rateLimiter = RateLimiter.create(rate); for (String stream : streams) { rateLimiter.acquire(); try { Await.result(client.release(stream)); System.out.println("Release ownership of stream " + stream); } catch (Exception e) { System.err.println("Failed to release ownership of stream " + stream); throw e; } } return 0; }
@Override public void run() { final DLSN dlsnToTruncate = prevDLSN; if (null == dlsnToTruncate) { return; } final Stopwatch stopwatch = Stopwatch.createStarted(); dlc.truncate(streamName, dlsnToTruncate).addEventListener( new FutureEventListener<Boolean>() { @Override public void onSuccess(Boolean value) { truncationStat.registerSuccessfulEvent(stopwatch.stop().elapsed(TimeUnit.MILLISECONDS)); } @Override public void onFailure(Throwable cause) { truncationStat.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MILLISECONDS)); LOG.error("Failed to truncate stream {} to {} : ", new Object[]{streamName, dlsnToTruncate, cause}); } }); }
private void doMoveStream(final String streamName) throws Exception { Await.result(srcClient.release(streamName).flatMap(new Function<Void, Future<Void>>() { @Override public Future<Void> apply(Void result) { return targetMonitor.check(streamName).addEventListener(new FutureEventListener<Void>() { @Override public void onSuccess(Void value) { logger.info("Moved stream {} from {} to {}.", new Object[]{streamName, source, target}); } @Override public void onFailure(Throwable cause) { logger.info("Failed to move stream {} from region {} to {} : ", new Object[]{streamName, source, target, cause}); } }); } })); }
synchronized void close() { if (null != client) { client.close(); } }
@Test(timeout = 20000) public void testFlushWhenBufferIsFull() throws Exception { DistributedLogClient client = mock(DistributedLogClient.class); when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())) .thenReturn(Future.value(new DLSN(1L, 1L, 999L))); ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder() .streams(Lists.newArrayList("stream1", "stream2")) .client(client) .compressionCodec(CompressionCodec.Type.LZ4) .firstSpeculativeTimeoutMs(100000) .maxSpeculativeTimeoutMs(200000) .speculativeBackoffMultiplier(2) .requestTimeoutMs(500000) .flushIntervalMs(0) .bufferSize(0) .scheduler(executorService) .build(); ByteBuffer buffer = ByteBuffer.wrap("test".getBytes(UTF_8)); writer.write(buffer); verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any()); writer.close(); }
Future<DLSN> write(KEY key, VALUE value) { int pid = partitioner.partition(key, streams.length); return client.write(streams[pid], ByteBuffer.wrap(value.toString().getBytes(UTF_8))); }
private void doMoveStream(final String stream, final Host from, final Host to) throws Exception { logger.info("Moving stream {} from {} to {}.", new Object[] { stream, from.address, to.address }); Await.result(from.getClient().release(stream).flatMap(new Function<Void, Future<Void>>() { @Override public Future<Void> apply(Void result) { logger.info("Released stream {} from {}.", stream, from.address); return to.getMonitor().check(stream).addEventListener(new FutureEventListener<Void>() { @Override public void onSuccess(Void value) { logger.info("Moved stream {} from {} to {}.", new Object[] { stream, from.address, to.address }); } @Override public void onFailure(Throwable cause) { logger.info("Failed to move stream {} from {} to {} : ", new Object[] { stream, from.address, to.address, cause }); } }); } })); }
protected void balanceFromSource(DistributedLogClientBuilder clientBuilder, ClusterBalancer balancer, String source, Optional<RateLimiter> rateLimiter) throws Exception { InetSocketAddress sourceAddr = DLSocketAddress.parseSocketAddress(source); DistributedLogClientBuilder sourceClientBuilder = DistributedLogClientBuilder.newBuilder(clientBuilder) .host(sourceAddr); Pair<DistributedLogClient, MonitorServiceClient> clientPair = ClientUtils.buildClient(sourceClientBuilder); try { Await.result(clientPair.getRight().setAcceptNewStream(false)); logger.info("Disable accepting new stream on proxy {}.", source); balancer.balanceAll(source, rebalanceConcurrency, rateLimiter); } finally { clientPair.getLeft().close(); } } }
throws Exception { DistributedLogClient client = mock(DistributedLogClient.class); when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())) .thenReturn(Future.value(new DLSN(1L, 1L, 999L))); ScheduledExecutorService executorService = ByteBuffer buffer1 = ByteBuffer.wrap(data); writer.write(buffer1); verify(client, times(0)).writeRecordSet((String) any(), (LogRecordSetBuffer) any()); LogRecordSet.Writer recordSetWriter1 = writer.getLogRecordSetWriter(); assertEquals(1, recordSetWriter1.getNumRecords()); verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any()); LogRecordSet.Writer recordSetWriter2 = writer.getLogRecordSetWriter(); assertEquals(1, recordSetWriter2.getNumRecords());
public Future<DLSN> write(KEY key, VALUE value) { int pid = partitioner.partition(key, streams.length); return client.write(streams[pid], ByteBuffer.wrap(value.toString().getBytes(UTF_8))); }