SpilledBufferOrEventSequence seq1 = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); SpilledBufferOrEventSequence seq2 = new SpilledBufferOrEventSequence(secondFile, secondChannel, buffer, pageSize); seq1.open(); BufferOrEvent next = seq1.getNext(); if (next.isEvent()) { BufferOrEvent expected = events1.get(numEvent++); assertNull(seq1.getNext()); assertEquals(events1.size(), numEvent); seq2.open(); BufferOrEvent next = seq2.getNext(); if (next.isEvent()) { BufferOrEvent expected = events2.get(numEvent++); assertNull(seq2.getNext()); assertEquals(events2.size(), numEvent);
@Test public void testCleanup() { try { ByteBuffer data = ByteBuffer.allocate(157); data.order(ByteOrder.LITTLE_ENDIAN); FileUtils.writeCompletely(fileChannel, data); fileChannel.position(54); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); seq.cleanup(); assertFalse(fileChannel.isOpen()); assertFalse(tempFile.exists()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
next = currentBuffered.getNext(); if (next == null) { completeBufferedSequence();
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp()); long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L; CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(bytesBuffered) .setAlignmentDurationNanos(latestAlignmentDurationNanos); toNotifyOnCheckpoint.triggerCheckpointOnBarrier( checkpointMetaData, checkpointBarrier.getCheckpointOptions(), checkpointMetrics); } }
SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); BufferOrEvent next = seq.getNext(); if (next.isEvent()) { BufferOrEvent expected = events.get(numEvent++); assertNull(seq.getNext());
@Test public void testEventSequence() { try { final Random rnd = new Random(); final int numEvents = 3000; final int numberOfChannels = 1656; final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(numEvents); for (int i = 0; i < numEvents; i++) { events.add(generateAndWriteEvent(fileChannel, rnd, numberOfChannels)); } fileChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); int i = 0; BufferOrEvent boe; while ((boe = seq.getNext()) != null) { BufferOrEvent expected = events.get(i); assertTrue(boe.isEvent()); assertEquals(expected.getEvent(), boe.getEvent()); assertEquals(expected.getChannelIndex(), boe.getChannelIndex()); i++; } assertEquals(numEvents, i); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
fileChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); validateBuffer(seq.getNext(), 1672, 7); seq.getNext(); fail("should fail with an exception");
@Test public void testBufferSequence() { try { final Random rnd = new Random(); final long seed = rnd.nextLong(); final int numBuffers = 325; final int numberOfChannels = 671; rnd.setSeed(seed); for (int i = 0; i < numBuffers; i++) { writeBuffer(fileChannel, rnd.nextInt(pageSize) + 1, rnd.nextInt(numberOfChannels)); } fileChannel.position(0L); rnd.setSeed(seed); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); for (int i = 0; i < numBuffers; i++) { validateBuffer(seq.getNext(), rnd.nextInt(pageSize) + 1, rnd.nextInt(numberOfChannels)); } // should have no more data assertNull(seq.getNext()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testIncompleteHeaderOnFirstElement() { try { ByteBuffer buf = ByteBuffer.allocate(7); buf.order(ByteOrder.LITTLE_ENDIAN); FileUtils.writeCompletely(fileChannel, buf); fileChannel.position(0); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); try { seq.getNext(); fail("should fail with an exception"); } catch (IOException e) { // expected } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
@Test public void testEmptyChannel() { try { SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize); seq.open(); assertNull(seq.getNext()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }