@Override public void writeUnconditionally(T value) { CompletableFuture<Void> ack = new CompletableFuture<>(); ByteBuffer serialized = serializer.serialize(value); try { PendingEvent event = PendingEvent.withHeader(null, serialized, ack); log.trace("Unconditionally writing: {}", value); synchronized (lock) { out.write(event); out.flush(); } } catch (SegmentSealedException e) { throw new CorruptedStateException("Unexpected end of segment ", e); } Futures.getAndHandleExceptions(ack, RuntimeException::new); }
@Synchronized private List<PendingEvent> updateSegments(StreamSegments newSteamSegments, Consumer<Segment> segmentSealedCallBack) { currentSegments = newSteamSegments; createMissingWriters(segmentSealedCallBack, newSteamSegments.getDelegationToken()); List<PendingEvent> toResend = new ArrayList<>(); Iterator<Entry<Segment, SegmentOutputStream>> iter = writers.entrySet().iterator(); while (iter.hasNext()) { Entry<Segment, SegmentOutputStream> entry = iter.next(); if (!currentSegments.getSegments().contains(entry.getKey())) { SegmentOutputStream writer = entry.getValue(); log.info("Closing writer {} on segment {} during segment refresh", writer, entry.getKey()); iter.remove(); try { writer.close(); } catch (SegmentSealedException e) { log.info("Caught segment sealed while refreshing on segment {}", entry.getKey()); } toResend.addAll(writer.getUnackedEventsOnSeal()); } } return toResend; }
private boolean flushInternal() { boolean success = true; for (SegmentOutputStream writer : selector.getWriters()) { try { writer.flush(); } catch (SegmentSealedException e) { // Segment sealed exception observed during a flush. Re-run flush on all the // available writers. success = false; log.warn("Flush on segment {} failed due to {}, it will be retried.", writer.getSegmentName(), e.getMessage()); } } return success; }
@Override public void write(byte[] b, int off, int len) throws IOException { ByteBuffer data = ByteBuffer.wrap(b, off, len); out.write(PendingEvent.withoutHeader(null, data, null)); }
@Override public void close() throws IOException { out.close(); meta.close(); }
@Override public void flush() throws IOException { out.flush(); }
@Synchronized private List<PendingEvent> removeAllWriters() { //get all pending events. List<PendingEvent> pendingEvents = new ArrayList<>(); writers.values().forEach(out -> { pendingEvents.addAll(out.getUnackedEventsOnSeal()); }); // remove all writers. writers.clear(); return pendingEvents; }
@Override public void write(ByteBuffer src) throws IOException { out.write(PendingEvent.withoutHeader(null, src, null)); }
@Override public void closeAndSeal() throws IOException { out.close(); meta.sealSegment(); meta.close(); }
@Override public void flush() throws TxnFailedException { checkFailed(); try { out.flush(); synchronized (lock) { removeCompleted(); checkFailed(); } } catch (SegmentSealedException e) { throw new TxnFailedException(e); } }
@Synchronized private List<PendingEvent> updateSegmentsUponSealed(StreamSegmentsWithPredecessors successors, Segment sealedSegment, Consumer<Segment> segmentSealedCallback) { currentSegments = currentSegments.withReplacementRange(sealedSegment, successors); createMissingWriters(segmentSealedCallback, currentSegments.getDelegationToken()); log.debug("Fetch unacked events for segment: {}, and adding new segments {}", sealedSegment, currentSegments); return writers.remove(sealedSegment).getUnackedEventsOnSeal(); }
@Test(timeout = 5000) public void testFlush() throws TxnFailedException, SegmentSealedException { UUID uuid = UUID.randomUUID(); SegmentOutputStream outputStream = Mockito.mock(SegmentOutputStream.class); @Cleanup SegmentTransactionImpl<String> txn = new SegmentTransactionImpl<>(uuid, outputStream, new JavaSerializer<String>()); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { PendingEvent event = (PendingEvent) invocation.getArgument(0); event.getAckFuture().complete(null); return null; } }).when(outputStream).write(Mockito.any(PendingEvent.class)); txn.writeEvent("hi"); verify(outputStream).write(Mockito.any(PendingEvent.class)); txn.flush(); verify(outputStream).flush(); Mockito.verifyNoMoreInteractions(outputStream); }
@GuardedBy("writeSealLock") private void resend(List<PendingEvent> toResend) { while (!toResend.isEmpty()) { List<PendingEvent> unsent = new ArrayList<>(); boolean sendFailed = false; log.info("Resending {} events", toResend.size()); for (PendingEvent event : toResend) { if (sendFailed) { unsent.add(event); } else { SegmentOutputStream segmentWriter = selector.getSegmentOutputStreamForKey(event.getRoutingKey()); if (segmentWriter == null) { log.info("No writer for segment during resend."); unsent.addAll(selector.refreshSegmentEventWriters(segmentSealedCallBack)); sendFailed = true; } else { segmentWriter.write(event); } } } toResend = unsent; } }
@Override public void close() throws TxnFailedException { flush(); try { out.close(); } catch (SegmentSealedException e) { throw new TxnFailedException(e); } }
when(s0Writer.getUnackedEventsOnSeal()) .thenReturn(ImmutableList.of(PendingEvent.withHeader("0", ByteBuffer.wrap("e".getBytes()), writerFuture)));
out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(testString.getBytes()), new CompletableFuture<>())); out.flush(); out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>())); out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>())); out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[150000]), new CompletableFuture<>())); assertEquals(in.read().capacity(), 15); assertEquals(in.read().capacity(), 15);
private ByteBuffer writeInt(SegmentOutputStream stream, int value) throws SegmentSealedException { ByteBuffer buffer = ByteBuffer.allocate(4).putInt(value); buffer.flip(); stream.write(PendingEvent.withHeader(null, buffer, new CompletableFuture<Void>())); return buffer; }
@Override public void close() { synchronized (lock) { try { out.close(); } catch (SegmentSealedException e) { log.warn("Error closing segment writer {}", out); } conditional.close(); meta.close(); in.close(); } } }
@Override public void writeEvent(Type event) throws TxnFailedException { checkFailed(); ByteBuffer buffer = serializer.serialize(event); CompletableFuture<Void> ack = new CompletableFuture<Void>(); PendingEvent pendingEvent = PendingEvent.withHeader(null, buffer, ack); synchronized (lock) { out.write(pendingEvent); outstanding.addLast(ack); removeCompleted(); } checkFailed(); }
@Override public void close() { if (closed.getAndSet(true)) { return; } pinger.close(); synchronized (writeFlushLock) { boolean success = false; while (!success) { success = true; for (SegmentOutputStream writer : selector.getWriters()) { try { writer.close(); } catch (SegmentSealedException e) { // Segment sealed exception observed during a close. Re-run close on all the available writers. success = false; log.warn("Close failed due to {}, it will be retried.", e.getMessage()); } } } } ExecutorServiceHelpers.shutdown(retransmitPool); }