public LlapAwareMemoryManager(Configuration conf) { super(conf); maxLoad = OrcConf.MEMORY_POOL.getDouble(conf); long memPerExecutor = LlapDaemonInfo.INSTANCE.getMemoryPerExecutor(); totalMemoryPool = (long) (memPerExecutor * maxLoad); if (LOG.isDebugEnabled()) { LOG.debug("Using LLAP memory manager for orc writer. memPerExecutor: {} maxLoad: {} totalMemPool: {}", LlapUtil.humanReadableByteCount(memPerExecutor), maxLoad, LlapUtil.humanReadableByteCount(totalMemoryPool)); } }
public double getDouble(Configuration conf) { return getDouble(null, conf); }
/** * Create the memory manager. * @param conf use the configuration to find the maximum size of the memory * pool. */ public MemoryManagerImpl(Configuration conf) { double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf); ROWS_BETWEEN_CHECKS = OrcConf.ROWS_BETWEEN_CHECKS.getLong(conf); LOG.info(OrcConf.ROWS_BETWEEN_CHECKS.getAttribute() + "=" + ROWS_BETWEEN_CHECKS); if(ROWS_BETWEEN_CHECKS < 1 || ROWS_BETWEEN_CHECKS > 10000) { throw new IllegalArgumentException(OrcConf.ROWS_BETWEEN_CHECKS.getAttribute() + "=" + ROWS_BETWEEN_CHECKS + " is outside valid range [1,10000]."); } totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean(). getHeapMemoryUsage().getMax() * maxLoad); ownerLock.lock(); }
protected WriterOptions(Properties tableProperties, Configuration conf) { configuration = conf; memoryManagerValue = getStaticMemoryManager(conf); stripeSizeValue = org.apache.orc.OrcConf.STRIPE_SIZE.getLong(tableProperties, conf); blockSizeValue = org.apache.orc.OrcConf.BLOCK_SIZE.getLong(tableProperties, conf); rowIndexStrideValue = (int) org.apache.orc.OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf); bufferSizeValue = (int) org.apache.orc.OrcConf.BUFFER_SIZE.getLong(tableProperties, conf); blockPaddingValue = org.apache.orc.OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(org.apache.orc.OrcConf.COMPRESS.getString(tableProperties, conf)); String versionName = org.apache.orc.OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); String enString = org.apache.orc.OrcConf.ENCODING_STRATEGY.getString(tableProperties, conf); encodingStrategy = EncodingStrategy.valueOf(enString); String compString = org.apache.orc.OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf); compressionStrategy = CompressionStrategy.valueOf(compString); paddingTolerance = org.apache.orc.OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf); bloomFilterColumns = org.apache.orc.OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties, conf); bloomFilterFpp = org.apache.orc.OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties, conf); }
protected WriterOptions(Properties tableProperties, Configuration conf) { configuration = conf; memoryManagerValue = getStaticMemoryManager(conf); stripeSizeValue = org.apache.orc.OrcConf.STRIPE_SIZE.getLong(tableProperties, conf); blockSizeValue = org.apache.orc.OrcConf.BLOCK_SIZE.getLong(tableProperties, conf); rowIndexStrideValue = (int) org.apache.orc.OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf); bufferSizeValue = (int) org.apache.orc.OrcConf.BUFFER_SIZE.getLong(tableProperties, conf); blockPaddingValue = org.apache.orc.OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(org.apache.orc.OrcConf.COMPRESS.getString(tableProperties, conf)); String versionName = org.apache.orc.OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); String enString = org.apache.orc.OrcConf.ENCODING_STRATEGY.getString(tableProperties, conf); encodingStrategy = EncodingStrategy.valueOf(enString); String compString = org.apache.orc.OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf); compressionStrategy = CompressionStrategy.valueOf(compString); paddingTolerance = org.apache.orc.OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf); bloomFilterColumns = org.apache.orc.OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties, conf); bloomFilterFpp = org.apache.orc.OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties, conf); }
StringBaseTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable) throws IOException { super(columnId, schema, writer, nullable); this.isDirectV2 = isNewWriteFormat(writer); stringOutput = writer.createStream(id, OrcProto.Stream.Kind.DICTIONARY_DATA); lengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer); rowOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.DATA), false, isDirectV2, writer); recordPosition(rowIndexPosition); rowIndexValueCount.add(0L); buildIndex = writer.buildIndex(); directStreamOutput = writer.createStream(id, OrcProto.Stream.Kind.DATA); directLengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer); Configuration conf = writer.getConfiguration(); dictionaryKeySizeThreshold = org.apache.orc.OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getDouble(conf); strideDictionaryCheck = org.apache.orc.OrcConf.ROW_INDEX_STRIDE_DICTIONARY_CHECK.getBoolean(conf); doneDictionaryCheck = false; }
StringBaseTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable) throws IOException { super(columnId, schema, writer, nullable); this.isDirectV2 = isNewWriteFormat(writer); stringOutput = writer.createStream(id, OrcProto.Stream.Kind.DICTIONARY_DATA); lengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer); rowOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.DATA), false, isDirectV2, writer); recordPosition(rowIndexPosition); rowIndexValueCount.add(0L); buildIndex = writer.buildIndex(); directStreamOutput = writer.createStream(id, OrcProto.Stream.Kind.DATA); directLengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer); Configuration conf = writer.getConfiguration(); dictionaryKeySizeThreshold = org.apache.orc.OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getDouble(conf); strideDictionaryCheck = org.apache.orc.OrcConf.ROW_INDEX_STRIDE_DICTIONARY_CHECK.getBoolean(conf); doneDictionaryCheck = false; }
/** * This function builds the options for the ORC Writer based on the JobConf. * @param conf the job configuration * @return a new options object */ public static OrcFile.WriterOptions buildOptions(Configuration conf) { return OrcFile.writerOptions(conf) .version(OrcFile.Version.byName(OrcConf.WRITE_FORMAT.getString(conf))) .setSchema(TypeDescription.fromString(OrcConf.MAPRED_OUTPUT_SCHEMA .getString(conf))) .compress(CompressionKind.valueOf(OrcConf.COMPRESS.getString(conf))) .encodingStrategy(OrcFile.EncodingStrategy.valueOf (OrcConf.ENCODING_STRATEGY.getString(conf))) .bloomFilterColumns(OrcConf.BLOOM_FILTER_COLUMNS.getString(conf)) .bloomFilterFpp(OrcConf.BLOOM_FILTER_FPP.getDouble(conf)) .blockSize(OrcConf.BLOCK_SIZE.getLong(conf)) .blockPadding(OrcConf.BLOCK_PADDING.getBoolean(conf)) .stripeSize(OrcConf.STRIPE_SIZE.getLong(conf)) .rowIndexStride((int) OrcConf.ROW_INDEX_STRIDE.getLong(conf)) .bufferSize((int) OrcConf.BUFFER_SIZE.getLong(conf)) .paddingTolerance(OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(conf)); }
opts.getDirectEncodingColumns(), opts.getSchema()); dictionaryKeySizeThreshold = OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getDouble(conf); if (callback != null) { callbackContext = new OrcFile.WriterContext(){