static LongOutputStream createLengthOutputStream(CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { if (orcEncoding == DWRF) { return new LongOutputStreamV1(compression, bufferSize, false, LENGTH); } else { return new LongOutputStreamV2(compression, bufferSize, false, LENGTH); } }
private void writeValues(EncodingType encoding) { if (numLiterals == 0) { return; } switch (encoding) { case SHORT_REPEAT: writeShortRepeatValues(); break; case DIRECT: writeDirectValues(); break; case PATCHED_BASE: writePatchedBaseValues(); break; default: writeDeltaValues(); break; } clearEncoder(); }
initializeLiterals(value); return; writeValues(determineEncoding()); writeValues(determineEncoding()); writeValues(EncodingType.SHORT_REPEAT); writeValues(EncodingType.DELTA); initializeLiterals(value); writeValues(determineEncoding());
public void flush() { if (numLiterals == 0) { return; } if (variableRunLength != 0) { writeValues(determineEncoding()); return; } if (fixedRunLength == 0) { throw new IllegalStateException("literals does not agree with run length counters"); } if (fixedRunLength < MIN_REPEAT) { variableRunLength = fixedRunLength; fixedRunLength = 0; writeValues(determineEncoding()); return; } if (fixedRunLength >= MIN_REPEAT && fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) { writeValues(EncodingType.SHORT_REPEAT); return; } isFixedDelta = true; writeValues(EncodingType.DELTA); }
@Override protected void writeValue(LongOutputStreamV2 outputStream, Long value) { outputStream.writeLong(value); }
@Override public void reset() { clearEncoder(); closed = false; buffer.reset(); checkpoints.clear(); }
@Override public void close() { closed = true; flush(); buffer.close(); }
private void writePatchedBaseValues() preparePatchedBlob();
public void flush() { if (numLiterals == 0) { return; } if (variableRunLength != 0) { writeValues(determineEncoding()); return; } if (fixedRunLength == 0) { throw new IllegalStateException("literals does not agree with run length counters"); } if (fixedRunLength < MIN_REPEAT) { variableRunLength = fixedRunLength; fixedRunLength = 0; writeValues(determineEncoding()); return; } if (fixedRunLength >= MIN_REPEAT && fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) { writeValues(EncodingType.SHORT_REPEAT); return; } isFixedDelta = true; writeValues(EncodingType.DELTA); }
@Override protected void writeValue(LongOutputStreamV2 outputStream, Long value) { outputStream.writeLong(value); }
@Override public void reset() { clearEncoder(); closed = false; buffer.reset(); checkpoints.clear(); }
@Override public void close() { closed = true; flush(); buffer.close(); }
private void writePatchedBaseValues() preparePatchedBlob();
private void writeValues(EncodingType encoding) { if (numLiterals == 0) { return; } switch (encoding) { case SHORT_REPEAT: writeShortRepeatValues(); break; case DIRECT: writeDirectValues(); break; case PATCHED_BASE: writePatchedBaseValues(); break; default: writeDeltaValues(); break; } clearEncoder(); }
@Override protected LongOutputStreamV2 createValueOutputStream() { return new LongOutputStreamV2(SNAPPY, COMPRESSION_BLOCK_SIZE, true, DATA); }
initializeLiterals(value); return; writeValues(determineEncoding()); writeValues(determineEncoding()); writeValues(EncodingType.SHORT_REPEAT); writeValues(EncodingType.DELTA); initializeLiterals(value); writeValues(determineEncoding());
public TimestampColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DateTimeZone hiveStorageTimeZone) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.secondsStream = new LongOutputStreamV1(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV1(compression, bufferSize, false, SECONDARY); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.secondsStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); this.nanosStream = new LongOutputStreamV2(compression, bufferSize, false, SECONDARY); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.baseTimestampInSeconds = new DateTime(2015, 1, 1, 0, 0, requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null")).getMillis() / MILLIS_PER_SECOND; }
public LongColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, Supplier<LongValueStatisticsBuilder> statisticsBuilderSupplier) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; if (orcEncoding == DWRF) { this.columnEncoding = new ColumnEncoding(DIRECT, 0); this.dataStream = new LongOutputStreamDwrf(compression, bufferSize, true, DATA); } else { this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new LongOutputStreamV2(compression, bufferSize, true, DATA); } this.presentStream = new PresentOutputStream(compression, bufferSize); this.statisticsBuilderSupplier = requireNonNull(statisticsBuilderSupplier, "statisticsBuilderSupplier is null"); this.statisticsBuilder = statisticsBuilderSupplier.get(); }
public DecimalColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { checkArgument(column >= 0, "column is negative"); checkArgument(orcEncoding != DWRF, "DWRF does not support %s type", type); this.column = column; this.type = (DecimalType) requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new DecimalOutputStream(compression, bufferSize); this.scaleStream = new LongOutputStreamV2(compression, bufferSize, true, SECONDARY); this.presentStream = new PresentOutputStream(compression, bufferSize); if (this.type.isShort()) { shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); } else { longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }
public SliceDictionaryColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding, DataSize stringStatisticsLimit) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compression = requireNonNull(compression, "compression is null"); this.bufferSize = bufferSize; this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.stringStatisticsLimitInBytes = toIntExact(requireNonNull(stringStatisticsLimit, "stringStatisticsLimit is null").toBytes()); LongOutputStream result; if (orcEncoding == DWRF) { result = new LongOutputStreamV1(compression, bufferSize, false, DATA); } else { result = new LongOutputStreamV2(compression, bufferSize, false, DATA); } this.dataStream = result; this.presentStream = new PresentOutputStream(compression, bufferSize); this.dictionaryDataStream = new ByteArrayOutputStream(compression, bufferSize, StreamKind.DICTIONARY_DATA); this.dictionaryLengthStream = createLengthOutputStream(compression, bufferSize, orcEncoding); values = new IntBigArray(); this.statisticsBuilder = newStringStatisticsBuilder(); }