final String columnNameDelimiter = table.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? table .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA); String compressType = OrcConf.COMPRESS.getString(table, conf);
public String getString(Configuration conf) { return getString(null, conf); }
@Override public void configure(JobConf conf) { if (value == null) { TypeDescription schema = TypeDescription.fromString(OrcConf.MAPRED_SHUFFLE_VALUE_SCHEMA .getString(conf)); value = OrcStruct.createValue(schema); } } }
private static CompressionKind getCompression(Properties schema, JobConf configuration, OrcEncoding orcEncoding) { String compressionName = OrcConf.COMPRESS.getString(schema, configuration); if (compressionName == null) { return CompressionKind.ZLIB; } CompressionKind compression; try { compression = CompressionKind.valueOf(compressionName.toUpperCase(ENGLISH)); } catch (IllegalArgumentException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Unknown " + orcEncoding + " compression type " + compressionName); } return compression; } }
@Override public void configure(JobConf conf) { if (key == null) { TypeDescription schema = TypeDescription.fromString(OrcConf.MAPRED_SHUFFLE_KEY_SCHEMA .getString(conf)); key = OrcStruct.createValue(schema); } }
protected WriterOptions(Properties tableProperties, Configuration conf) { configuration = conf; memoryManagerValue = getStaticMemoryManager(conf); stripeSizeValue = org.apache.orc.OrcConf.STRIPE_SIZE.getLong(tableProperties, conf); blockSizeValue = org.apache.orc.OrcConf.BLOCK_SIZE.getLong(tableProperties, conf); rowIndexStrideValue = (int) org.apache.orc.OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf); bufferSizeValue = (int) org.apache.orc.OrcConf.BUFFER_SIZE.getLong(tableProperties, conf); blockPaddingValue = org.apache.orc.OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(org.apache.orc.OrcConf.COMPRESS.getString(tableProperties, conf)); String versionName = org.apache.orc.OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); String enString = org.apache.orc.OrcConf.ENCODING_STRATEGY.getString(tableProperties, conf); encodingStrategy = EncodingStrategy.valueOf(enString); String compString = org.apache.orc.OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf); compressionStrategy = CompressionStrategy.valueOf(compString); paddingTolerance = org.apache.orc.OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf); bloomFilterColumns = org.apache.orc.OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties, conf); bloomFilterFpp = org.apache.orc.OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties, conf); }
protected WriterOptions(Properties tableProperties, Configuration conf) { configuration = conf; memoryManagerValue = getStaticMemoryManager(conf); stripeSizeValue = org.apache.orc.OrcConf.STRIPE_SIZE.getLong(tableProperties, conf); blockSizeValue = org.apache.orc.OrcConf.BLOCK_SIZE.getLong(tableProperties, conf); rowIndexStrideValue = (int) org.apache.orc.OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf); bufferSizeValue = (int) org.apache.orc.OrcConf.BUFFER_SIZE.getLong(tableProperties, conf); blockPaddingValue = org.apache.orc.OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(org.apache.orc.OrcConf.COMPRESS.getString(tableProperties, conf)); String versionName = org.apache.orc.OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); String enString = org.apache.orc.OrcConf.ENCODING_STRATEGY.getString(tableProperties, conf); encodingStrategy = EncodingStrategy.valueOf(enString); String compString = org.apache.orc.OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf); compressionStrategy = CompressionStrategy.valueOf(compString); paddingTolerance = org.apache.orc.OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf); bloomFilterColumns = org.apache.orc.OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties, conf); bloomFilterFpp = org.apache.orc.OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties, conf); }
OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf); compressValue = CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties, conf).toUpperCase()); enforceBufferSize = OrcConf.ENFORCE_COMPRESSION_BUFFER_SIZE.getBoolean(tableProperties, conf); String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties, conf); versionValue = Version.byName(versionName); String enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties, conf); encodingStrategy = EncodingStrategy.valueOf(enString); OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf); compressionStrategy = CompressionStrategy.valueOf(compString); bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties, conf); bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties, bloomFilterVersion = BloomFilterVersion.fromString( OrcConf.BLOOM_FILTER_WRITE_VERSION.getString(tableProperties, conf)); shims = HadoopShimsFactory.get(); writeVariableLengthBlocks = OrcConf.WRITE_VARIABLE_LENGTH_BLOCKS.getBoolean(tableProperties,conf); directEncodingColumns = OrcConf.DIRECT_ENCODING_COLUMNS.getString( tableProperties, conf);
long length) { TypeDescription schema = TypeDescription.fromString(OrcConf.MAPRED_INPUT_SCHEMA.getString(conf)); Reader.Options options = reader.options() .range(start, length) OrcConf.INCLUDE_COLUMNS.getString(conf))); String kryoSarg = OrcConf.KRYO_SARG.getString(conf); String sargColumns = OrcConf.SARG_COLUMNS.getString(conf); if (kryoSarg != null && sargColumns != null) { byte[] sargBytes = Base64.decodeBase64(kryoSarg);
/** * This function builds the options for the ORC Writer based on the JobConf. * @param conf the job configuration * @return a new options object */ public static OrcFile.WriterOptions buildOptions(Configuration conf) { return OrcFile.writerOptions(conf) .version(OrcFile.Version.byName(OrcConf.WRITE_FORMAT.getString(conf))) .setSchema(TypeDescription.fromString(OrcConf.MAPRED_OUTPUT_SCHEMA .getString(conf))) .compress(CompressionKind.valueOf(OrcConf.COMPRESS.getString(conf))) .encodingStrategy(OrcFile.EncodingStrategy.valueOf (OrcConf.ENCODING_STRATEGY.getString(conf))) .bloomFilterColumns(OrcConf.BLOOM_FILTER_COLUMNS.getString(conf)) .bloomFilterFpp(OrcConf.BLOOM_FILTER_FPP.getDouble(conf)) .blockSize(OrcConf.BLOCK_SIZE.getLong(conf)) .blockPadding(OrcConf.BLOCK_PADDING.getBoolean(conf)) .stripeSize(OrcConf.STRIPE_SIZE.getLong(conf)) .rowIndexStride((int) OrcConf.ROW_INDEX_STRIDE.getLong(conf)) .bufferSize((int) OrcConf.BUFFER_SIZE.getLong(conf)) .paddingTolerance(OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(conf)); }