private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(inMemoryTable, unSafeComparator); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); this.sort(inMemoryTable); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
if (chunk.isMemory()) { if (LOG.isDebugEnabled()) { debug(LOG, "Remove intermediate memory tuples: " + chunk.getMemoryTuples().usedMem());
if (chunk.isMemory()) { if (LOG.isDebugEnabled()) { debug(LOG, "Remove intermediate memory tuples: " + chunk.getMemoryTuples().usedMem());
inMemoryTable.addTuple(tuple); if (inMemoryTable.usedMem() > sortBufferBytesNum) { // if input data exceeds main-memory at least once long runEndTime = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " run loading time: " + (runEndTime - runStartTime) + " msec"); runStartTime = runEndTime; info(LOG, "Memory consumption exceeds " + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false));
inMemoryTable.addTuple(tuple); if (inMemoryTable.usedMem() > sortBufferBytesNum) { // if input data exceeds main-memory at least once long runEndTime = System.currentTimeMillis(); info(LOG, "Chunk #" + chunkId + " run loading time: " + (runEndTime - runStartTime) + " msec"); runStartTime = runEndTime; info(LOG, "Memory consumption exceeds " + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false));
assertEquals(0, unSafeTupleList.usedMem()); assertEquals(0, unSafeTupleList.size()); int tupleSize = unSafeTupleList.usedMem(); assertEquals(1, unSafeTupleList.size()); assertEquals(tuple, unSafeTupleList.get(0)); assertEquals(0, unSafeTupleList.usedMem()); assertEquals(0, unSafeTupleList.size()); assertEquals(tupleSize * testCount, unSafeTupleList.usedMem());