static LongOutputStream createLengthOutputStream(CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { if (orcEncoding == DWRF) { return new LongOutputStreamV1(compression, bufferSize, false, LENGTH); } else { return new LongOutputStreamV2(compression, bufferSize, false, LENGTH); } }
@Override protected LongOutputStreamV2 createValueOutputStream() { return new LongOutputStreamV2(SNAPPY, COMPRESSION_BLOCK_SIZE, true, DATA); }
public TimestampColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DateTimeZone hiveStorageTimeZone) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.secondsStream = new LongOutputStreamV1(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV1(compression, bufferSize, false, SECONDARY); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.secondsStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV2(compression, bufferSize, false, SECONDARY); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.baseTimestampInSeconds = new DateTime(2015, 1, 1, 0, 0, requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null")).getMillis() / MILLIS_PER_SECOND; }
public LongColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, Supplier<LongValueStatisticsBuilder> statisticsBuilderSupplier) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.dataStream = new LongOutputStreamDwrf(compression, bufferSize, true, DATA); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.statisticsBuilderSupplier = requireNonNull(statisticsBuilderSupplier, "statisticsBuilderSupplier is null"); this.statisticsBuilder = statisticsBuilderSupplier.get(); }
public SliceDictionaryColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DataSize stringStatisticsLimit) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compression = requireNonNull(compression, "compression is null"); this.bufferSize = bufferSize; this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.stringStatisticsLimitInBytes = toIntExact(requireNonNull(stringStatisticsLimit, "stringStatisticsLimit is null").toBytes()); LongOutputStream result; if (orcEncoding == DWRF) { result = new LongOutputStreamV1(compression, bufferSize, false, DATA); } else { result = new LongOutputStreamV2(compression, bufferSize, false, DATA); } this.dataStream = result; this.presentStream = new PresentOutputStream(compression, bufferSize); this.dictionaryDataStream = new ByteArrayOutputStream(compression, bufferSize, StreamKind.DICTIONARY_DATA); this.dictionaryLengthStream = createLengthOutputStream(compression, bufferSize, orcEncoding); values = new IntBigArray(); this.statisticsBuilder = newStringStatisticsBuilder(); }
public DecimalColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { checkArgument(column >= 0, "column is negative"); checkArgument(orcEncoding != DWRF, "DWRF does not support %s type", type); this.column = column; this.type = (DecimalType) requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new DecimalOutputStream(compression, bufferSize); this.scaleStream = new LongOutputStreamV2(compression, bufferSize, true, SECONDARY); this.presentStream = new PresentOutputStream(compression, bufferSize); if (this.type.isShort()) { shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); } else { longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }
static LongOutputStream createLengthOutputStream(CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { if (orcEncoding == DWRF) { return new LongOutputStreamV1(compression, bufferSize, false, LENGTH); } else { return new LongOutputStreamV2(compression, bufferSize, false, LENGTH); } }
@Override protected LongOutputStreamV2 createValueOutputStream() { return new LongOutputStreamV2(SNAPPY, COMPRESSION_BLOCK_SIZE, true, DATA); }
public TimestampColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DateTimeZone hiveStorageTimeZone) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.secondsStream = new LongOutputStreamV1(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV1(compression, bufferSize, false, SECONDARY); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.secondsStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV2(compression, bufferSize, false, SECONDARY); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.baseTimestampInSeconds = new DateTime(2015, 1, 1, 0, 0, requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null")).getMillis() / MILLIS_PER_SECOND; }
public LongColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, Supplier<LongValueStatisticsBuilder> statisticsBuilderSupplier) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.dataStream = new LongOutputStreamDwrf(compression, bufferSize, true, DATA); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.statisticsBuilderSupplier = requireNonNull(statisticsBuilderSupplier, "statisticsBuilderSupplier is null"); this.statisticsBuilder = statisticsBuilderSupplier.get(); }
public SliceDictionaryColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DataSize stringStatisticsLimit) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compression = requireNonNull(compression, "compression is null"); this.bufferSize = bufferSize; this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.stringStatisticsLimitInBytes = toIntExact(requireNonNull(stringStatisticsLimit, "stringStatisticsLimit is null").toBytes()); LongOutputStream result; if (orcEncoding == DWRF) { result = new LongOutputStreamV1(compression, bufferSize, false, DATA); } else { result = new LongOutputStreamV2(compression, bufferSize, false, DATA); } this.dataStream = result; this.presentStream = new PresentOutputStream(compression, bufferSize); this.dictionaryDataStream = new ByteArrayOutputStream(compression, bufferSize, StreamKind.DICTIONARY_DATA); this.dictionaryLengthStream = createLengthOutputStream(compression, bufferSize, orcEncoding); values = new IntBigArray(); this.statisticsBuilder = newStringStatisticsBuilder(); }
public DecimalColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { checkArgument(column >= 0, "column is negative"); checkArgument(orcEncoding != DWRF, "DWRF does not support %s type", type); this.column = column; this.type = (DecimalType) requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new DecimalOutputStream(compression, bufferSize); this.scaleStream = new LongOutputStreamV2(compression, bufferSize, true, SECONDARY); this.presentStream = new PresentOutputStream(compression, bufferSize); if (this.type.isShort()) { shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); } else { longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }