/** * Allocates some unpooled memory and creates a new memory segment that represents * that memory. * * <p>This method is similar to {@link #allocateUnpooledSegment(int, Object)}, but the * memory segment will have null as the owner. * * @param size The size of the memory segment to allocate. * @return A new memory segment, backed by unpooled heap memory. */ public static MemorySegment allocateUnpooledSegment(int size) { return allocateUnpooledSegment(size, null); }
private static List<MemorySegment> getMemory(int numSegments, int segmentSize) { ArrayList<MemorySegment> list = new ArrayList<MemorySegment>(numSegments); for (int i = 0; i < numSegments; i++) { list.add(MemorySegmentFactory.allocateUnpooledSegment(segmentSize)); } return list; }
public MemorySegment setupNormalizedKeysMemSegment(T[] data, int normKeyLen, TypeComparator<T> comparator) { MemorySegment memSeg = MemorySegmentFactory.allocateUnpooledSegment(2048); // Setup normalized Keys in the memory segment int offset = 0; for (T e : data) { comparator.putNormalizedKey(e, memSeg, offset, normKeyLen); offset += normKeyLen; } return memSeg; }
MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);
private static BufferOrEvent createBuffer(int channel, int pageSize) { final int size = sizeCounter++; byte[] bytes = new byte[size]; RND.nextBytes(bytes); MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(pageSize); memory.put(0, bytes); Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE); buf.setSize(size); // retain an additional time so it does not get disposed after being read by the input gate buf.retainBuffer(); return new BufferOrEvent(buf, channel); }
private static BufferOrEvent createBuffer(int channel, int size) { byte[] bytes = new byte[size]; RND.nextBytes(bytes); MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE); memory.put(0, bytes); Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE); buf.setSize(size); // retain an additional time so it does not get disposed after being read by the input gate buf.retainBuffer(); return new BufferOrEvent(buf, channel); }
public static BufferOrEvent generateRandomBuffer(int size, int channelIndex) { MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE); for (int i = 0; i < size; i++) { seg.put(i, (byte) i); } Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE); buf.setSize(size); return new BufferOrEvent(buf, channelIndex); }
MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(randomInt.length());
@Override MemorySegment allocateNewSegment(Object owner) { return MemorySegmentFactory.allocateUnpooledSegment(segmentSize, owner); }
@Override MemorySegment allocateNewSegment(Object owner) { return MemorySegmentFactory.allocateUnpooledSegment(segmentSize, owner); }
/** * Allocates some unpooled memory and creates a new memory segment that represents * that memory. * * <p>This method is similar to {@link #allocateUnpooledSegment(int, Object)}, but the * memory segment will have null as the owner. * * @param size The size of the memory segment to allocate. * @return A new memory segment, backed by unpooled heap memory. */ public static MemorySegment allocateUnpooledSegment(int size) { return allocateUnpooledSegment(size, null); }
/** * Allocates some unpooled memory and creates a new memory segment that represents * that memory. * * <p>This method is similar to {@link #allocateUnpooledSegment(int, Object)}, but the * memory segment will have null as the owner. * * @param size The size of the memory segment to allocate. * @return A new memory segment, backed by unpooled heap memory. */ public static MemorySegment allocateUnpooledSegment(int size) { return allocateUnpooledSegment(size, null); }
@Override MemorySegment allocateNewSegment(Object owner) { return MemorySegmentFactory.allocateUnpooledSegment(segmentSize, owner); }
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException { BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader( ioManager, id, new LinkedBlockingQueue<>(), compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize); return new HeaderlessChannelReaderInputView(inReader, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), blockCount, lastSegmentLimit, false); }
SpillReadBufferPool(int numberOfBuffers, int memorySegmentSize) { this.buffers = new ArrayDeque<>(numberOfBuffers); synchronized (buffers) { for (int i = 0; i < numberOfBuffers; i++) { buffers.add(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(memorySegmentSize), this)); } } }
private void createFreeMemoryIfNeeded() { if (freeMemory == null && rowInSpill(beginRow)) { // Only initialize freeMemory when we need to read spilled records. freeMemory = new ArrayList<>(); // Iterator will use memory segments from heap for (int i = 0; i < READ_BUFFER; i++) { freeMemory.add(MemorySegmentFactory.allocateUnpooledSegment(segmentSize)); } } }
public FixedLengthBufferPool(int numberOfBuffers, int memorySegmentSize, MemoryType memoryType) { checkArgument(numberOfBuffers > 0, "There should be at least one buffer."); this.numberOfBuffers = numberOfBuffers; this.memorySegmentSize = memorySegmentSize; this.freeOnDestroy = true; this.memorySegments = new ArrayDeque<>(numberOfBuffers); synchronized (memorySegments) { if (memoryType == MemoryType.HEAP) { for (int i = 0; i < numberOfBuffers; i++) { memorySegments.add(MemorySegmentFactory.allocateUnpooledSegment(memorySegmentSize)); } } else if (memoryType == MemoryType.OFF_HEAP) { for (int i = 0; i < numberOfBuffers; i++) { ByteBuffer memory = ByteBuffer.allocateDirect(memorySegmentSize); memorySegments.add(MemorySegmentFactory.wrapPooledOffHeapMemory(memory, null)); } } else { throw new IllegalArgumentException("Unknown memory type " + memoryType); } } }
public static AbstractChannelWriterOutputView createOutputView( IOManager ioManager, FileIOChannel.ID channel, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel); return new CompressedHeaderlessChannelWriterOutputView( bufferWriter, compressionCodecFactory, compressionBlockSize); } else { BlockChannelWriter<MemorySegment> blockWriter = ioManager.createBlockChannelWriter(channel); return new HeaderlessChannelWriterOutputView( blockWriter, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), segmentSize); } }
MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);
public static AbstractChannelReaderInputView createInputView( IOManager ioManager, ChannelWithMeta channel, List<FileIOChannel> channels, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { CompressedHeaderlessChannelReaderInputView in = new CompressedHeaderlessChannelReaderInputView( channel.getChannel(), ioManager, compressionCodecFactory, compressionBlockSize, channel.getBlockCount()); channels.add(in.getReader()); return in; } else { BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel.getChannel()); channels.add(reader); return new HeaderlessChannelReaderInputView( reader, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), channel.getBlockCount(), channel.getNumBytesInLastBlock(), false); } }