int getEstimatedBufferSize(String colNames, int bs) { long availableMem = getMemoryAvailableForORC(); if (colNames != null) { final int numCols = colNames.split(",").length; if (numCols > COLUMN_COUNT_THRESHOLD) { // In BufferedStream, there are 3 outstream buffers (compressed, // uncompressed and overflow) and list of previously compressed buffers. // Since overflow buffer is rarely used, lets consider only 2 allocation. // Also, initially, the list of compression buffers will be empty. final int outStreamBuffers = codec == null ? 1 : 2; // max possible streams per column is 5. For string columns, there is // ROW_INDEX, PRESENT, DATA, LENGTH, DICTIONARY_DATA streams. final int maxStreams = 5; // Lets assume 10% memory for holding dictionary in memory and other // object allocations final long miscAllocation = (long) (0.1f * availableMem); // compute the available memory final long remainingMem = availableMem - miscAllocation; int estBufferSize = (int) (remainingMem / (maxStreams * outStreamBuffers * numCols)); estBufferSize = getClosestBufferSize(estBufferSize, bs); if (estBufferSize > bs) { estBufferSize = bs; } LOG.info("WIDE TABLE - Number of columns: " + numCols + " Chosen compression buffer size: " + estBufferSize); return estBufferSize; } } return bs; }
int getEstimatedBufferSize(String colNames, int bs) { long availableMem = getMemoryAvailableForORC(); if (colNames != null) { final int numCols = colNames.split(",").length; if (numCols > COLUMN_COUNT_THRESHOLD) { // In BufferedStream, there are 3 outstream buffers (compressed, // uncompressed and overflow) and list of previously compressed buffers. // Since overflow buffer is rarely used, lets consider only 2 allocation. // Also, initially, the list of compression buffers will be empty. final int outStreamBuffers = codec == null ? 1 : 2; // max possible streams per column is 5. For string columns, there is // ROW_INDEX, PRESENT, DATA, LENGTH, DICTIONARY_DATA streams. final int maxStreams = 5; // Lets assume 10% memory for holding dictionary in memory and other // object allocations final long miscAllocation = (long) (0.1f * availableMem); // compute the available memory final long remainingMem = availableMem - miscAllocation; int estBufferSize = (int) (remainingMem / (maxStreams * outStreamBuffers * numCols)); estBufferSize = getClosestBufferSize(estBufferSize, bs); if (estBufferSize > bs) { estBufferSize = bs; } LOG.info("WIDE TABLE - Number of columns: " + numCols + " Chosen compression buffer size: " + estBufferSize); return estBufferSize; } } return bs; }