@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
/** * Return the peak memory used so far, in bytes. */ long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
private void updatePeakMemoryUsed() { long mem = getMemoryUsage(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } }
@Test public void testSortingEmptyInput() { final ShuffleInMemorySorter sorter = new ShuffleInMemorySorter( consumer, 100, shouldUseRadixSort()); final ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
@Test(expected=IllegalStateException.class) public void mustCallWriteBeforeSuccessfulStop() throws IOException { createWriter(false).stop(true); }
@Test(expected=PandaException.class) public void writeFailurePropagates() throws Exception { class BadRecords extends scala.collection.AbstractIterator<Product2<Object, Object>> { @Override public boolean hasNext() { throw new PandaException(); } @Override public Product2<Object, Object> next() { return null; } } final UnsafeShuffleWriter<Object, Object> writer = createWriter(true); writer.write(new BadRecords()); }
@Override public PackedRecordPointer newKey() { return new PackedRecordPointer(); }
/** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
private void updatePeakMemoryUsed() { // sorter can be null if this writer is closed if (sorter != null) { long mem = sorter.getPeakMemoryUsedBytes(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } } }
@Test public void testSortingEmptyInput() { final ShuffleInMemorySorter sorter = new ShuffleInMemorySorter( consumer, 100, shouldUseRadixSort()); final ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
@Test(expected=IllegalStateException.class) public void mustCallWriteBeforeSuccessfulStop() throws IOException { createWriter(false).stop(true); }
/** * Return the peak memory used so far, in bytes. */ long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
/** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
private void updatePeakMemoryUsed() { long mem = getMemoryUsage(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } }
private void updatePeakMemoryUsed() { // sorter can be null if this writer is closed if (sorter != null) { long mem = sorter.getPeakMemoryUsedBytes(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } } }
@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
@Test public void doNotNeedToCallWriteBeforeUnsuccessfulStop() throws IOException { createWriter(false).stop(false); }
/** * Return the peak memory used so far, in bytes. */ long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
/** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }