DFSPacket packet = new DFSPacket(packetBuf, stripedWriter.getMaxChunksPerPacket(), blockOffset4Target, seqNo4Target++, int ckLen = ((toWrite - 1) / stripedWriter.getBytesPerChecksum() + 1) packet.writeChecksum(stripedWriter.getChecksumBuf(), ckOff, ckLen); ckOff += ckLen; packet.writeData(targetBuffer, toWrite); packet.writeTo(targetOutputStream);
private void queueCurrentPacket() { synchronized (dataQueue) { if (currentPacket == null) return; currentPacket.addTraceParent(Trace.currentSpan()); dataQueue.addLast(currentPacket); lastQueuedSeqno = currentPacket.getSeqno(); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Queued packet " + currentPacket.getSeqno()); } currentPacket = null; dataQueue.notifyAll(); } }
@Override public String toString() { return "packet seqno: " + this.seqno + " offsetInBlock: " + this.offsetInBlock + " lastPacketInBlock: " + this.lastPacketInBlock + " lastByteOffsetInBlock: " + this.getLastByteOffsetBlock(); }
if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient writeChunk allocating new packet seqno=" + currentPacket.getSeqno() + ", src=" + src + ", packetSize=" + packetSize + currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); currentPacket.incNumChunks(); bytesCurBlock += len; if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || bytesCurBlock == blockSize) { if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient writeChunk packet full seqno=" + currentPacket.getSeqno() + ", src=" + src + ", bytesCurBlock=" + bytesCurBlock + currentPacket.setSyncBlock(shouldSyncBlock); waitAndQueueCurrentPacket(); bytesCurBlock = 0;
} else { long parents[] = one.getTraceParents(); if (parents.length > 0) { scope = Trace.startSpan("dataStreamer", new TraceInfo(0, parents[0])); long lastByteOffsetInBlock = one.getLastByteOffsetBlock(); if (lastByteOffsetInBlock > blockSize) { throw new IOException("BlockSize " + blockSize + if (one.isLastPacketInBlock()) { if (!one.isHeartbeatPacket()) { span = scope.detach(); one.setTraceSpan(span); dataQueue.removeFirst(); ackQueue.addLast(one); one.writeTo(blockStream); blockStream.flush(); } catch (IOException e) { long tmpBytesSent = one.getLastByteOffsetBlock(); if (bytesSent < tmpBytesSent) { bytesSent = tmpBytesSent; if (one.isLastPacketInBlock()) {
protected synchronized void writeChunk(ByteBuffer buffer, int len, byte[] checksum, int ckoff, int cklen) throws IOException { writeChunkPrepare(len, ckoff, cklen); currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(buffer, len); currentPacket.incNumChunks(); getStreamer().incBytesCurBlock(len); // If packet is full, enqueue it for transmission if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || getStreamer().getBytesCurBlock() == blockSize) { enqueueCurrentPacketFull(); } }
SpanId[] parents = one.getTraceParents(); if (parents.length > 0) { scope = dfsClient.getTracer(). long lastByteOffsetInBlock = one.getLastByteOffsetBlock(); if (lastByteOffsetInBlock > stat.getBlockSize()) { throw new IOException("BlockSize " + stat.getBlockSize() + if (one.isLastPacketInBlock()) { if (!one.isHeartbeatPacket()) { if (scope != null) { spanId = scope.getSpanId(); scope.detach(); one.setTraceScope(scope); packetSendTime.put(one.getSeqno(), Time.monotonicNow()); dataQueue.notifyAll(); one.writeTo(blockStream); blockStream.flush(); } catch (IOException e) { long tmpBytesSent = one.getLastByteOffsetBlock(); if (bytesSent < tmpBytesSent) { bytesSent = tmpBytesSent; if (one.isLastPacketInBlock()) {
@Test public void testAddParentsGetParents() throws Exception { DFSPacket p = new DFSPacket(null, maxChunksPerPacket, 0, 0, checksumSize, false); long parents[] = p.getTraceParents(); Assert.assertEquals(0, parents.length); p.addTraceParent(123); p.addTraceParent(123); parents = p.getTraceParents(); Assert.assertEquals(1, parents.length); Assert.assertEquals(123, parents[0]); parents = p.getTraceParents(); // test calling 'get' again. Assert.assertEquals(1, parents.length); Assert.assertEquals(123, parents[0]); p.addTraceParent(1); p.addTraceParent(456); p.addTraceParent(789); parents = p.getTraceParents(); Assert.assertEquals(4, parents.length); Assert.assertEquals(1, parents[0]); Assert.assertEquals(123, parents[1]); Assert.assertEquals(456, parents[2]); Assert.assertEquals(789, parents[3]); } }
one = ackQueue.getFirst(); if (one.getSeqno() != seqno) { throw new IOException("ResponseProcessor: Expecting seqno " + " for block " + block + one.getSeqno() + " but received " + seqno); isLastPacketInBlock = one.isLastPacketInBlock(); block.setNumBytes(one.getLastByteOffsetBlock()); scope = Trace.continueSpan(one.getTraceSpan()); one.setTraceSpan(null); lastAckedSeqno = seqno; pipelineRecoveryCount = 0; dataQueue.notifyAll(); one.releaseBuffer(byteArrayManager);
/** * For heartbeat packets, create buffer directly by new byte[] * since heartbeats should not be blocked. */ private DFSPacket createHeartbeatPacket() { final byte[] buf = new byte[PacketHeader.PKT_MAX_HEADER_LEN]; return new DFSPacket(buf, 0, 0, DFSPacket.HEART_BEAT_SEQNO, 0, false); }
one = ackQueue.getFirst(); if (one.getSeqno() != seqno) { throw new IOException("ResponseProcessor: Expecting seqno " + " for block " + block + one.getSeqno() + " but received " + seqno); isLastPacketInBlock = one.isLastPacketInBlock(); block.setNumBytes(one.getLastByteOffsetBlock()); scope = one.getTraceScope(); if (scope != null) { scope.reattach(); one.setTraceScope(null); dataQueue.notifyAll(); one.releaseBuffer(byteArrayManager);
} else if (currentPacket != null) { currentPacket.releaseBuffer(byteArrayManager); currentPacket = null; currentPacket.setSyncBlock(isSync); waitAndQueueCurrentPacket(); currentPacket.setSyncBlock(shouldSyncBlock || isSync); waitAndQueueCurrentPacket(); bytesCurBlock = 0;
public synchronized void writeData(ByteBuffer inBuffer, int len) throws ClosedChannelException { checkBuffer(); len = len > inBuffer.remaining() ? inBuffer.remaining() : len; if (dataPos + len > buf.length) { throw new BufferOverflowException(); } for (int i = 0; i < len; i++) { buf[dataPos + i] = inBuffer.get(); } dataPos += len; }
/** create an empty packet to mark the end of the block. */ void setCurrentPacketToEmpty() throws InterruptedIOException { currentPacket = createPacket(0, 0, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), true); currentPacket.setSyncBlock(shouldSyncBlock); }
/** * Add a trace parent span for this packet. * <p> * Trace parent spans for a packet are the trace spans responsible for * adding data to that packet. We store them as an array of longs for * efficiency. * <p> * Protected by the DFSOutputStream dataQueue lock. */ public void addTraceParent(Span span) { if (span == null) { return; } addTraceParent(span.getSpanId()); }
if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient writeChunk allocating new packet seqno=" + currentPacket.getSeqno() + ", src=" + src + ", packetSize=" + packetSize + currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); currentPacket.incNumChunks(); bytesCurBlock += len; if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || bytesCurBlock == blockSize) { if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient writeChunk packet full seqno=" + currentPacket.getSeqno() + ", src=" + src + ", bytesCurBlock=" + bytesCurBlock + currentPacket.setSyncBlock(shouldSyncBlock); waitAndQueueCurrentPacket(); bytesCurBlock = 0;
} else { long parents[] = one.getTraceParents(); if (parents.length > 0) { scope = Trace.startSpan("dataStreamer", new TraceInfo(0, parents[0])); long lastByteOffsetInBlock = one.getLastByteOffsetBlock(); if (lastByteOffsetInBlock > blockSize) { throw new IOException("BlockSize " + blockSize + if (one.isLastPacketInBlock()) { if (!one.isHeartbeatPacket()) { span = scope.detach(); one.setTraceSpan(span); dataQueue.removeFirst(); ackQueue.addLast(one); one.writeTo(blockStream); blockStream.flush(); } catch (IOException e) { long tmpBytesSent = one.getLastByteOffsetBlock(); if (bytesSent < tmpBytesSent) { bytesSent = tmpBytesSent; if (one.isLastPacketInBlock()) {
@Override protected synchronized void writeChunk(byte[] b, int offset, int len, byte[] checksum, int ckoff, int cklen) throws IOException { writeChunkPrepare(len, ckoff, cklen); currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); currentPacket.incNumChunks(); getStreamer().incBytesCurBlock(len); // If packet is full, enqueue it for transmission if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || getStreamer().getBytesCurBlock() == blockSize) { enqueueCurrentPacketFull(); } }
@Test public void testAddParentsGetParents() throws Exception { DFSPacket p = new DFSPacket(null, maxChunksPerPacket, 0, 0, checksumSize, false); SpanId parents[] = p.getTraceParents(); Assert.assertEquals(0, parents.length); p.addTraceParent(new SpanId(0, 123)); p.addTraceParent(new SpanId(0, 123)); parents = p.getTraceParents(); Assert.assertEquals(1, parents.length); Assert.assertEquals(new SpanId(0, 123), parents[0]); parents = p.getTraceParents(); // test calling 'get' again. Assert.assertEquals(1, parents.length); Assert.assertEquals(new SpanId(0, 123), parents[0]); p.addTraceParent(new SpanId(0, 1)); p.addTraceParent(new SpanId(0, 456)); p.addTraceParent(new SpanId(0, 789)); parents = p.getTraceParents(); Assert.assertEquals(4, parents.length); Assert.assertEquals(new SpanId(0, 1), parents[0]); Assert.assertEquals(new SpanId(0, 123), parents[1]); Assert.assertEquals(new SpanId(0, 456), parents[2]); Assert.assertEquals(new SpanId(0, 789), parents[3]); } }