private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); this.sort(inMemoryTable); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(inMemoryTable, unSafeComparator); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
/** * Entry method. * * @param list * @param schema input schema * @param sortSpecs sort specs * @param comp comparator for Tim sort * @return a sorted list of tuples */ public static List<UnSafeTuple> sort(QueryContext queryContext, UnSafeTupleList list, Schema schema, SortSpec[] sortSpecs, Comparator<UnSafeTuple> comp) { UnSafeTuple[] in = list.toArray(new UnSafeTuple[list.size()]); RadixSortContext context = new RadixSortContext(in, schema, sortSpecs, comp, queryContext.getInt(SessionVars.TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT)); long before = System.currentTimeMillis(); recursiveCallForNextKey(context, 0, context.in.length, 0); context.msdRadixSortTime += System.currentTimeMillis() - before; context.printStat(); ListIterator<UnSafeTuple> it = list.listIterator(); for (UnSafeTuple t : context.in) { it.next(); it.set(t); } return list; }
/** * Sort a tuple block and store them into a chunk file */ private Chunk sortAndStoreChunk(int chunkId, UnSafeTupleList tupleBlock) throws IOException { int rowNum = tupleBlock.size(); long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(tupleBlock, unSafeComparator); long sortEnd = System.currentTimeMillis(); long chunkWriteStart = System.currentTimeMillis(); Path outputPath = getChunkPathForWrite(0, chunkId); final DirectRawFileWriter appender = new DirectRawFileWriter(context.getConf(), null, inSchema, intermediateMeta, outputPath); appender.init(); for (Tuple t : tupleBlock) { appender.addTuple(t); } appender.close(); long chunkWriteEnd = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " sort and written (" + FileUtil.humanReadableByteCount(appender.getOffset(), false) + " bytes, " + rowNum + " rows, " + "sort time: " + (sortEnd - sortStart) + " msec, " + "write time: " + (chunkWriteEnd - chunkWriteStart) + " msec)"); FileFragment frag = new FileFragment("", outputPath, 0, new File(localFS.makeQualified(outputPath).toUri()).length()); return new Chunk(inSchema, frag, intermediateMeta); }
/** * Sort a tuple block and store them into a chunk file */ private Chunk sortAndStoreChunk(int chunkId, UnSafeTupleList tupleBlock) throws IOException { int rowNum = tupleBlock.size(); long sortStart = System.currentTimeMillis(); this.sort(tupleBlock); long sortEnd = System.currentTimeMillis(); long chunkWriteStart = System.currentTimeMillis(); Path outputPath = getChunkPathForWrite(0, chunkId); final DirectRawFileWriter appender = new DirectRawFileWriter(context.getConf(), null, inSchema, intermediateMeta, outputPath); appender.init(); for (Tuple t : tupleBlock) { appender.addTuple(t); } appender.close(); long chunkWriteEnd = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " sort and written (" + FileUtil.humanReadableByteCount(appender.getOffset(), false) + " bytes, " + rowNum + " rows, " + "sort time: " + (sortEnd - sortStart) + " msec, " + "write time: " + (chunkWriteEnd - chunkWriteStart) + " msec)"); FileFragment frag = new FileFragment("", outputPath, 0, new File(localFS.makeQualified(outputPath).toUri()).length()); return new Chunk(inSchema, frag, intermediateMeta); }
if(inMemoryTable.size() > 0) { //if there are at least one or more input tuples
@Test public void testSort() { Comparator<UnSafeTuple> comparator = new UnSafeComparator(schema, sortSpecs); RadixSort.sort(queryContext, tuples, schema, sortSpecs, comparator); IntStream.range(0, tuples.size() - 1) .forEach(i -> { assertTrue(tuples.get(i) + " precedes " + tuples.get(i + 1) + " at " + i, comparator.compare(tuples.get(i), tuples.get(i + 1)) <= 0); }); } }
if(inMemoryTable.size() > 0) { //if there are at least one or more input tuples
UnSafeTupleList unSafeTupleList = new UnSafeTupleList(SchemaUtil.toDataTypes(schema), 100, StorageUnit.KB); assertEquals(0, unSafeTupleList.usedMem()); assertEquals(0, unSafeTupleList.size()); assertEquals(1, unSafeTupleList.size()); assertEquals(tuple, unSafeTupleList.get(0)); assertEquals(0, unSafeTupleList.size()); assertEquals(testCount, unSafeTupleList.size()); assertEquals(tupleSize * testCount, unSafeTupleList.usedMem());