public boolean addTuple(Tuple tuple) { int prevPos = currentRowBlock.getMemory().writerPosition(); if (currentRowBlock.getWriter().addTuple(tuple)) { UnSafeTuple unSafeTuple = new UnSafeTuple(); unSafeTuple.set(currentRowBlock.getMemory(), prevPos, dataTypes); return super.add(unSafeTuple); } else { this.totalUsedMem += currentRowBlock.usedMem(); this.currentRowBlock = new MemoryRowBlock(dataTypes, new FixedSizeLimitSpec(pageSize), true); this.rowBlocks.add(currentRowBlock); return this.addTuple(tuple); } }
/** * Release the cached pages */ public void release() { for (MemoryRowBlock rowBlock : rowBlocks) { rowBlock.release(); } super.clear(); rowBlocks.clear(); totalUsedMem = 0; }
public OffHeapRowBlockReader(MemoryRowBlock rowBlock) { this(rowBlock.getMemory(), rowBlock.getDataTypes(), rowBlock.rows()); }
MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema)); String[] lines = explainStr.split("\n"); try { for (String line : lines) { rowBlock.getWriter().startRow(); rowBlock.getWriter().putText(line); rowBlock.getWriter().endRow(); MemoryBlock memoryBlock = rowBlock.getMemory(); ByteBuffer uncompressed = memoryBlock.getBuffer().nioBuffer(0, memoryBlock.readableBytes()); int uncompressedLength = uncompressed.remaining(); serializedResBuilder.setSerializedTuples(ByteString.copyFrom(uncompressed)); serializedResBuilder.setSchema(schema.getProto()); serializedResBuilder.setRows(rowBlock.rows()); } finally { rowBlock.release();
MemoryRowBlock rowBlock = partitionMemoryMap.get(partId); if (rowBlock == null) { rowBlock = new MemoryRowBlock(dataTypes, initialBufferSize, true, plan.getStorageType()); partitionMemoryMap.put(partId, rowBlock); totalBufferCapacity += rowBlock.capacity(); RowWriter writer = rowBlock.getWriter(); long prevUsedMem = rowBlock.usedMem(); totalBufferCapacity -= rowBlock.capacity(); totalBufferCapacity += rowBlock.capacity(); // calculate resizeable buffer capacity usedBufferSize += (rowBlock.usedMem() - prevUsedMem);
} else { if (rowBlock == null) { rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(tableDesc.getLogicalSchema())); rowBlock.getWriter().addTuple(currentTuple); currentRow++; rowCount++; MemoryBlock memoryBlock = rowBlock.getMemory(); ByteBuffer rows = memoryBlock.getBuffer().nioBuffer(0, memoryBlock.readableBytes()); rowBlock.clear();
final SettableFuture<MemoryRowBlock> future = SettableFuture.create(); if (rowBlock == null) { rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(scanNode.getOutSchema())); rowBlock.clear(); future.set(rowBlock); return future; rowBlock.clear(); int endRow = currentNumRows + fetchRowNum; while (currentNumRows < endRow) { break; } else { rowBlock.getWriter().addTuple(tuple); currentNumRows++; if (currentNumRows >= maxRow) { if (rowBlock.rows() > 0) { totalRows += rowBlock.rows();
public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.KB * 128); long allocatedEnd = System.currentTimeMillis(); LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); for (int i = 0; i < rowNum; i++) { fillRow(i, rowBlock.getWriter()); } long writeEnd = System.currentTimeMillis(); LOG.info("writing takes " + (writeEnd - writeStart) + " msec"); return rowBlock; }
@Override public void run() { try { rowBlock.clear(); int endRow = currentNumRows + fetchRowNum; while (currentNumRows < endRow) { Tuple tuple = scanExec.next(); if (tuple == null) { eof = true; break; } else { rowBlock.getWriter().addTuple(tuple); currentNumRows++; if (currentNumRows >= maxRow) { eof = true; break; } } } if (rowBlock.rows() > 0) { totalRows += rowBlock.rows(); } future.set(rowBlock); } catch (Throwable t) { future.setException(t); } } });
public UnSafeTupleList(DataType[] dataTypes, int initialArraySize, int pageSize) { super(initialArraySize); this.dataTypes = dataTypes; this.pageSize = pageSize; this.rowBlocks = Lists.newArrayList(); this.currentRowBlock = new MemoryRowBlock(dataTypes, new FixedSizeLimitSpec(pageSize), true); this.rowBlocks.add(currentRowBlock); }
public static Tuple[] sortToArray(MemoryRowBlock rowBlock, Comparator<Tuple> comparator) { Tuple[] tuples = new Tuple[rowBlock.rows()]; ZeroCopyTuple zcTuple; if(rowBlock.getMemory().hasAddress()) { zcTuple = new UnSafeTuple(); } else { zcTuple = new HeapTuple(); } RowBlockReader reader = rowBlock.getReader(); for (int i = 0; i < rowBlock.rows() && reader.next(zcTuple); i++) { tuples[i] = zcTuple; if(rowBlock.getMemory().hasAddress()) { zcTuple = new UnSafeTuple(); } else { zcTuple = new HeapTuple(); } } Arrays.sort(tuples, comparator); return tuples; }
@Override public long getOffset() throws IOException { return hasExternalBuf ? pos : pos + rowBlock.getMemory().writerPosition(); }
final SettableFuture<MemoryRowBlock> future = SettableFuture.create(); if (rowBlock == null) { rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(scanNode.getOutSchema())); rowBlock.clear(); future.set(rowBlock); return future;
public static List<Tuple> sort(MemoryRowBlock rowBlock, Comparator<Tuple> comparator) { List<Tuple> tupleList = Lists.newArrayList(); ZeroCopyTuple zcTuple; if(rowBlock.getMemory().hasAddress()) { zcTuple = new UnSafeTuple(); } else { zcTuple = new HeapTuple(); } RowBlockReader reader = rowBlock.getReader(); while(reader.next(zcTuple)) { tupleList.add(zcTuple); if(rowBlock.getMemory().hasAddress()) { zcTuple = new UnSafeTuple(); } else { zcTuple = new HeapTuple(); } } Collections.sort(tupleList, comparator); return tupleList; }