@Override public boolean add(UnSafeTuple tuple) { return addTuple(tuple); }
/** * Release and reset */ @Override public void clear() { release(); this.currentRowBlock = new MemoryRowBlock(dataTypes, new FixedSizeLimitSpec(pageSize), true); this.rowBlocks.add(currentRowBlock); } }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); this.sort(inMemoryTable); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
/** * Entry method. * * @param list * @param schema input schema * @param sortSpecs sort specs * @param comp comparator for Tim sort * @return a sorted list of tuples */ public static List<UnSafeTuple> sort(QueryContext queryContext, UnSafeTupleList list, Schema schema, SortSpec[] sortSpecs, Comparator<UnSafeTuple> comp) { UnSafeTuple[] in = list.toArray(new UnSafeTuple[list.size()]); RadixSortContext context = new RadixSortContext(in, schema, sortSpecs, comp, queryContext.getInt(SessionVars.TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT)); long before = System.currentTimeMillis(); recursiveCallForNextKey(context, 0, context.in.length, 0); context.msdRadixSortTime += System.currentTimeMillis() - before; context.printStat(); ListIterator<UnSafeTuple> it = list.listIterator(); for (UnSafeTuple t : context.in) { it.next(); it.set(t); } return list; }
UnSafeTupleList unSafeTupleList = new UnSafeTupleList(SchemaUtil.toDataTypes(schema), 100, StorageUnit.KB); assertEquals(0, unSafeTupleList.usedMem()); assertEquals(0, unSafeTupleList.size()); unSafeTupleList.addTuple(tuple); int tupleSize = unSafeTupleList.usedMem(); assertEquals(1, unSafeTupleList.size()); assertEquals(tuple, unSafeTupleList.get(0)); unSafeTupleList.clear(); assertEquals(0, unSafeTupleList.usedMem()); assertEquals(0, unSafeTupleList.size()); DatumFactory.createFloat8(rnd.nextDouble())}; tuples[i] = new VTuple(datums); unSafeTupleList.addTuple(tuples[i]); assertEquals(testCount, unSafeTupleList.size()); assertEquals(tupleSize * testCount, unSafeTupleList.usedMem()); assertEquals(tuples[i], unSafeTupleList.get(i)); unSafeTupleList.release();
inMemoryTable.addTuple(tuple); if (inMemoryTable.usedMem() > sortBufferBytesNum) { // if input data exceeds main-memory at least once long runEndTime = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " run loading time: " + (runEndTime - runStartTime) + " msec"); runStartTime = runEndTime; info(LOG, "Memory consumption exceeds " + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false)); inMemoryTable.clear(); chunkId++; if(inMemoryTable.size() > 0) { //if there are at least one or more input tuples
if (chunk.isMemory()) { if (LOG.isDebugEnabled()) { debug(LOG, "Remove intermediate memory tuples: " + chunk.getMemoryTuples().usedMem()); chunk.getMemoryTuples().release(); } else if (chunk.getFragment().getInputSourceId().contains(INTERMEDIATE_FILE_PREFIX)) { localFS.delete(chunk.getFragment().getPath(), true);
@Before public void setup() { List<DataType> dataTypeList = schema.getRootColumns().stream().map(Column::getDataType).collect(Collectors.toList()); tuples = new UnSafeTupleList(dataTypeList.toArray(new DataType[dataTypeList.size()]), tupleNum); tuples.addTuple(tuple); }); tuples.addTuple(tuple); makeMinTuple(tuple); tuples.addTuple(tuple); makeNullTuple(tuple); tuples.addTuple(tuple); makeMaxTuple(tuple); tuples.addTuple(tuple); makeMinTuple(tuple); tuples.addTuple(tuple); makeNullTuple(tuple); tuples.addTuple(tuple);
@Test public void testSort() { Comparator<UnSafeTuple> comparator = new UnSafeComparator(schema, sortSpecs); RadixSort.sort(queryContext, tuples, schema, sortSpecs, comparator); IntStream.range(0, tuples.size() - 1) .forEach(i -> { assertTrue(tuples.get(i) + " precedes " + tuples.get(i + 1) + " at " + i, comparator.compare(tuples.get(i), tuples.get(i + 1)) <= 0); }); } }
@Override public void init() throws IOException { if(allocatedCoreNum > 1) { this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum); } this.sortTmpDir = getExecutorTmpDir(); int initialArraySize = context.getQueryContext().getInt(SessionVars.SORT_LIST_SIZE); this.inMemoryTable = new UnSafeTupleList(SchemaUtil.toDataTypes(inSchema), initialArraySize); this.unSafeComparator = new UnSafeComparator(inSchema, sortSpecs); this.primitiveComparator = new PrimitiveComparator(inSchema, sortSpecs); super.init(); }
/** * Sort a tuple block and store them into a chunk file */ private Chunk sortAndStoreChunk(int chunkId, UnSafeTupleList tupleBlock) throws IOException { int rowNum = tupleBlock.size(); long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(tupleBlock, unSafeComparator); long sortEnd = System.currentTimeMillis(); long chunkWriteStart = System.currentTimeMillis(); Path outputPath = getChunkPathForWrite(0, chunkId); final DirectRawFileWriter appender = new DirectRawFileWriter(context.getConf(), null, inSchema, intermediateMeta, outputPath); appender.init(); for (Tuple t : tupleBlock) { appender.addTuple(t); } appender.close(); long chunkWriteEnd = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " sort and written (" + FileUtil.humanReadableByteCount(appender.getOffset(), false) + " bytes, " + rowNum + " rows, " + "sort time: " + (sortEnd - sortStart) + " msec, " + "write time: " + (chunkWriteEnd - chunkWriteStart) + " msec)"); FileFragment frag = new FileFragment("", outputPath, 0, new File(localFS.makeQualified(outputPath).toUri()).length()); return new Chunk(inSchema, frag, intermediateMeta); }
inMemoryTable.addTuple(tuple); if (inMemoryTable.usedMem() > sortBufferBytesNum) { // if input data exceeds main-memory at least once long runEndTime = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " run loading time: " + (runEndTime - runStartTime) + " msec"); runStartTime = runEndTime; info(LOG, "Memory consumption exceeds " + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false)); inMemoryTable.clear(); chunkId++; if(inMemoryTable.size() > 0) { //if there are at least one or more input tuples
if (chunk.isMemory()) { if (LOG.isDebugEnabled()) { debug(LOG, "Remove intermediate memory tuples: " + chunk.getMemoryTuples().usedMem()); chunk.getMemoryTuples().release(); } else if (chunk.getFragment().getTableName().contains(INTERMEDIATE_FILE_PREFIX)) { localFS.delete(chunk.getFragment().getPath(), true);
@Override public void init() throws IOException { if(allocatedCoreNum > 1) { this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum); } this.sortTmpDir = getExecutorTmpDir(); int initialArraySize = context.getQueryContext().getInt(SessionVars.SORT_LIST_SIZE); this.inMemoryTable = new UnSafeTupleList(SchemaUtil.toDataTypes(inSchema), initialArraySize); this.unSafeComparator = new UnSafeComparator(inSchema, sortSpecs); this.primitiveComparator = new PrimitiveComparator(inSchema, sortSpecs); super.init(); }
/** * Sort a tuple block and store them into a chunk file */ private Chunk sortAndStoreChunk(int chunkId, UnSafeTupleList tupleBlock) throws IOException { int rowNum = tupleBlock.size(); long sortStart = System.currentTimeMillis(); this.sort(tupleBlock); long sortEnd = System.currentTimeMillis(); long chunkWriteStart = System.currentTimeMillis(); Path outputPath = getChunkPathForWrite(0, chunkId); final DirectRawFileWriter appender = new DirectRawFileWriter(context.getConf(), null, inSchema, intermediateMeta, outputPath); appender.init(); for (Tuple t : tupleBlock) { appender.addTuple(t); } appender.close(); long chunkWriteEnd = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " sort and written (" + FileUtil.humanReadableByteCount(appender.getOffset(), false) + " bytes, " + rowNum + " rows, " + "sort time: " + (sortEnd - sortStart) + " msec, " + "write time: " + (chunkWriteEnd - chunkWriteStart) + " msec)"); FileFragment frag = new FileFragment("", outputPath, 0, new File(localFS.makeQualified(outputPath).toUri()).length()); return new Chunk(inSchema, frag, intermediateMeta); }
/** * Release and reset */ @Override public void clear() { release(); this.currentRowBlock = new MemoryRowBlock(dataTypes, new FixedSizeLimitSpec(pageSize), true); this.rowBlocks.add(currentRowBlock); } }
@Override public boolean add(UnSafeTuple tuple) { return addTuple(tuple); }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(inMemoryTable, unSafeComparator); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
@After public void teardown() { tuples.release(); }
public boolean addTuple(Tuple tuple) { int prevPos = currentRowBlock.getMemory().writerPosition(); if (currentRowBlock.getWriter().addTuple(tuple)) { UnSafeTuple unSafeTuple = new UnSafeTuple(); unSafeTuple.set(currentRowBlock.getMemory(), prevPos, dataTypes); return super.add(unSafeTuple); } else { this.totalUsedMem += currentRowBlock.usedMem(); this.currentRowBlock = new MemoryRowBlock(dataTypes, new FixedSizeLimitSpec(pageSize), true); this.rowBlocks.add(currentRowBlock); return this.addTuple(tuple); } }