@Override public String toString() { return "init=" + FileUtil.humanReadableByteCount(initSize, false) + ",limit=" + FileUtil.humanReadableByteCount(limitBytes, false) + ",overflow_ratio=" + allowedOVerflowRatio + ",inc_ratio=" + incRatio; } }
@Override public String toString() { return "init=" + FileUtil.humanReadableByteCount(initSize, false) + ",limit=" + FileUtil.humanReadableByteCount(limitBytes, false) + ",overflow_ratio=" + allowedOVerflowRatio + ",inc_ratio=" + incRatio; } }
@Override public String toString() { return "memory=" + FileUtil.humanReadableByteCount(capacity(), false) + "," + limitSpec; } }
@Override public String toString() { return "memory=" + FileUtil.humanReadableByteCount(capacity(), false) + "," + limitSpec; } }
public int increasedSize(int currentSize) { if (currentSize < initSize) { return (int) initSize; } if (currentSize == Integer.MAX_VALUE) { throw new TajoRuntimeException(new UnsupportedException( "Current size already exceeds the maximum size (" + Integer.MAX_VALUE + " bytes)")); } long nextSize = (long) (currentSize + ((float) currentSize * incRatio)); if (nextSize > limitBytes) { LOG.info("Increasing reaches size limit (" + FileUtil.humanReadableByteCount(limitBytes, false) + ")"); nextSize = limitBytes; } if (nextSize > Integer.MAX_VALUE) { LOG.info("Increasing reaches maximum size (" + FileUtil.humanReadableByteCount(Integer.MAX_VALUE, false) + ")"); nextSize = Integer.MAX_VALUE; } return (int) nextSize; }
public int increasedSize(int currentSize) { if (currentSize < initSize) { return (int) initSize; } if (currentSize == Integer.MAX_VALUE) { throw new TajoRuntimeException(new UnsupportedException( "Current size already exceeds the maximum size (" + Integer.MAX_VALUE + " bytes)")); } long nextSize = (long) (currentSize + ((float) currentSize * incRatio)); if (nextSize > limitBytes) { LOG.info("Increasing reaches size limit (" + FileUtil.humanReadableByteCount(limitBytes, false) + ")"); nextSize = limitBytes; } if (nextSize > Integer.MAX_VALUE) { LOG.info("Increasing reaches maximum size (" + FileUtil.humanReadableByteCount(Integer.MAX_VALUE, false) + ")"); nextSize = Integer.MAX_VALUE; } return (int) nextSize; }
public static String tableStatToString(CatalogProtos.TableStatsProto tableStats) { if (tableStats == null) { return "No input statistics"; } String result = ""; result += "TotalBytes: " + FileUtil.humanReadableByteCount(tableStats.getNumBytes(), false) + " (" + tableStats.getNumBytes() + " B)"; result += ", ReadBytes: " + FileUtil.humanReadableByteCount(tableStats.getReadBytes(), false) + " (" + tableStats.getReadBytes() + " B)"; result += ", ReadRows: " + (tableStats.getNumRows() == 0 ? "-" : tableStats.getNumRows()); return result; }
public static String tableStatToString(CatalogProtos.TableStatsProto tableStats) { if (tableStats == null) { return "No input statistics"; } String result = ""; result += "TotalBytes: " + FileUtil.humanReadableByteCount(tableStats.getNumBytes(), false) + " (" + tableStats.getNumBytes() + " B)"; result += ", ReadBytes: " + FileUtil.humanReadableByteCount(tableStats.getReadBytes(), false) + " (" + tableStats.getReadBytes() + " B)"; result += ", ReadRows: " + (tableStats.getNumRows() == 0 ? "-" : tableStats.getNumRows()); return result; }
@Before public void printTestName() { /* protect a travis stalled build */ BufferPoolMXBean direct = BufferPool.getDirectBufferPool(); BufferPoolMXBean mapped = BufferPool.getMappedBufferPool(); System.out.println(String.format("Used heap: %s/%s, direct:%s/%s, mapped:%s/%s, Active Threads: %d, Run: %s.%s", FileUtil.humanReadableByteCount(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), false), FileUtil.humanReadableByteCount(Runtime.getRuntime().maxMemory(), false), FileUtil.humanReadableByteCount(direct.getMemoryUsed(), false), FileUtil.humanReadableByteCount(direct.getTotalCapacity(), false), FileUtil.humanReadableByteCount(mapped.getMemoryUsed(), false), FileUtil.humanReadableByteCount(mapped.getTotalCapacity(), false), Thread.activeCount(), getClass().getSimpleName(), name.getMethodName())); }
@Override public void ensureSize(int size) { if (!buffer.isWritable(size)) { int newBlockSize = limitSpec.increasedSize(size); if (!limitSpec.canIncrease(buffer.writableBytes() + newBlockSize)) { throw new ValueOutOfRangeException("Cannot increase RowBlock anymore."); } resize(newBlockSize); LOG.info("Increase DirectRowBlock to " + FileUtil.humanReadableByteCount(newBlockSize, false)); } }
@Override public void ensureSize(int size) { if (!buffer.isWritable(size)) { int newBlockSize = limitSpec.increasedSize(size); if (!limitSpec.canIncrease(buffer.writableBytes() + newBlockSize)) { throw new ValueOutOfRangeException("Cannot increase RowBlock anymore."); } resize(newBlockSize); LOG.info("Increase DirectRowBlock to " + FileUtil.humanReadableByteCount(newBlockSize, false)); } }
private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix, boolean endOfTuple) { TableStats stat = tableDesc.getStats(); String volume = stat == null ? (endOfTuple ? "0 B" : "unknown bytes") : FileUtil.humanReadableByteCount(stat.getNumBytes(), false); long resultRows = stat == null ? TajoConstants.UNKNOWN_ROW_NUMBER : stat.getNumRows(); String displayRowNum; if (resultRows == TajoConstants.UNKNOWN_ROW_NUMBER) { if (endOfTuple) { displayRowNum = totalPrintedRows + " rows"; } else { displayRowNum = "unknown row number"; } } else { displayRowNum = resultRows + " rows"; } return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")"; }
private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix, boolean endOfTuple) { TableStats stat = tableDesc.getStats(); String volume = stat == null ? (endOfTuple ? "0 B" : "unknown bytes") : FileUtil.humanReadableByteCount(stat.getNumBytes(), false); long resultRows = stat == null ? TajoConstants.UNKNOWN_ROW_NUMBER : stat.getNumRows(); String displayRowNum; if (resultRows == TajoConstants.UNKNOWN_ROW_NUMBER) { if (endOfTuple) { displayRowNum = totalPrintedRows + " rows"; } else { displayRowNum = "unknown row number"; } } else { displayRowNum = resultRows + " rows"; } return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")"; }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); OffHeapRowBlockUtils.sort(inMemoryTable, unSafeComparator); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
@Override public Chunk call() throws Exception { final Path outputPath = getChunkPathForWrite(level + 1, nextRunId); info(LOG, mergeFanout + " files are being merged to an output file " + outputPath.getName()); long mergeStartTime = System.currentTimeMillis(); final Scanner merger = createKWayMerger(inputFiles, startIdx, mergeFanout); merger.init(); final DirectRawFileWriter output = new DirectRawFileWriter(context.getConf(), null, inSchema, intermediateMeta, outputPath); output.init(); Tuple mergeTuple; while((mergeTuple = merger.next()) != null) { output.addTuple(mergeTuple); } merger.close(); output.close(); long mergeEndTime = System.currentTimeMillis(); info(LOG, outputPath.getName() + " is written to a disk. (" + FileUtil.humanReadableByteCount(output.getOffset(), false) + " bytes, " + (mergeEndTime - mergeStartTime) + " msec)"); File f = new File(localFS.makeQualified(outputPath).toUri()); FileFragment frag = new FileFragment(INTERMEDIATE_FILE_PREFIX + outputPath.getName(), outputPath, 0, f.length()); return new Chunk(inSchema, frag, intermediateMeta); } }
public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock, Path outputFile) throws IOException { DirectRawFileWriter writer = new DirectRawFileWriter(conf, null, schema, meta, outputFile); writer.init(); writer.writeRowBlock(rowBlock); writer.close(); FileStatus status = outputFile.getFileSystem(conf).getFileStatus(outputFile); assertTrue(status.getLen() > 0); LOG.info("Written file size: " + FileUtil.humanReadableByteCount(status.getLen(), false)); return status; }
private Scanner getScanner(Chunk chunk) throws IOException { if (chunk.isMemory()) { long sortStart = System.currentTimeMillis(); this.sort(inMemoryTable); Scanner scanner = new MemTableScanner<>(inMemoryTable, inMemoryTable.size(), inMemoryTable.usedMem()); if(LOG.isDebugEnabled()) { debug(LOG, "Memory Chunk sort (" + FileUtil.humanReadableByteCount(inMemoryTable.usedMem(), false) + " bytes, " + inMemoryTable.size() + " rows, sort time: " + (System.currentTimeMillis() - sortStart) + " msec)"); } return scanner; } else { return TablespaceManager.getLocalFs().getScanner(chunk.meta, chunk.schema, chunk.fragment, chunk.schema); } }
public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.KB * 128); long allocatedEnd = System.currentTimeMillis(); LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); for (int i = 0; i < rowNum; i++) { fillRow(i, rowBlock.getWriter()); } long writeEnd = System.currentTimeMillis(); LOG.info("writing takes " + (writeEnd - writeStart) + " msec"); return rowBlock; }
@VisibleForTesting public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, String [] lineage, long tableVolume, boolean left) throws IOException { boolean inMemoryInnerJoinFlag; QueryContext queryContext = context.getQueryContext(); if (queryContext.containsKey(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT)) { inMemoryInnerJoinFlag = tableVolume <= context.getQueryContext().getLong(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB; } else { inMemoryInnerJoinFlag = tableVolume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB; } LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), StringUtils.join(lineage), FileUtil.humanReadableByteCount(tableVolume, false), (inMemoryInnerJoinFlag ? "" : "not "))); return inMemoryInnerJoinFlag; }
@VisibleForTesting public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, String [] lineage, long tableVolume, boolean left) throws IOException { boolean inMemoryInnerJoinFlag; QueryContext queryContext = context.getQueryContext(); if (queryContext.containsKey(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT)) { inMemoryInnerJoinFlag = tableVolume <= context.getQueryContext().getLong(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB; } else { inMemoryInnerJoinFlag = tableVolume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB; } LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), StringUtils.join(lineage), FileUtil.humanReadableByteCount(tableVolume, false), (inMemoryInnerJoinFlag ? "" : "not "))); return inMemoryInnerJoinFlag; }