private void createBooleanOutputStream() { checkState(booleanOutputStream == null); booleanOutputStream = new BooleanOutputStream(buffer); for (int groupsCount : groupsCounts) { booleanOutputStream.writeBooleans(groupsCount, true); booleanOutputStream.recordCheckpoint(); } booleanOutputStream.writeBooleans(currentGroupCount, true); }
public void close() { closed = true; if (booleanOutputStream != null) { booleanOutputStream.close(); } }
public Optional<List<BooleanStreamCheckpoint>> getCheckpoints() { checkArgument(closed); if (booleanOutputStream == null) { return Optional.empty(); } return Optional.of(booleanOutputStream.getCheckpoints()); }
BooleanOutputStream output = new BooleanOutputStream(buffer); output.writeBooleans(count, true); output.recordCheckpoint(); output.close(); List<BooleanStreamCheckpoint> batchWriteCheckpoints = output.getCheckpoints(); DynamicSliceOutput slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice); output.reset(); for (int count : counts) { for (int i = 0; i < count; i++) { output.writeBoolean(true); output.recordCheckpoint(); output.close(); List<BooleanStreamCheckpoint> singleWriteCheckpoints = output.getCheckpoints(); slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice);
outputStream.reset(); outputStream.writeBooleans(32, true); expectedValues.addAll(Collections.nCopies(32, true)); outputStream.writeBooleans(32, false); expectedValues.addAll(Collections.nCopies(32, false)); outputStream.writeBooleans(1, true); expectedValues.add(true); outputStream.writeBooleans(1, false); expectedValues.add(false); outputStream.writeBooleans(34, true); expectedValues.addAll(Collections.nCopies(34, true)); outputStream.writeBooleans(34, false); expectedValues.addAll(Collections.nCopies(34, false)); outputStream.writeBoolean(true); expectedValues.add(true); outputStream.writeBoolean(false); expectedValues.add(false); outputStream.close(); StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(33); streamDataOutput.writeData(sliceOutput); Stream stream = streamDataOutput.getStream();
public void writeBoolean(boolean value) { checkArgument(!closed); if (!value && booleanOutputStream == null) { createBooleanOutputStream(); } if (booleanOutputStream != null) { booleanOutputStream.writeBoolean(value); } currentGroupCount++; }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override protected BooleanOutputStream createValueOutputStream() { return new BooleanOutputStream(SNAPPY, COMPRESSION_BLOCK_SIZE); }
public void recordCheckpoint() { checkArgument(!closed); groupsCounts.add(currentGroupCount); currentGroupCount = 0; if (booleanOutputStream != null) { booleanOutputStream.recordCheckpoint(); } }
@Override public void reset() { closed = false; dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); statisticsBuilder = new BooleanStatisticsBuilder(); } }
public void writeBoolean(boolean value) { checkState(!closed); if (value) { data |= 0x1 << (7 - bitsInData); } bitsInData++; if (bitsInData == 8) { flushData(); } }
public long getBufferedBytes() { if (booleanOutputStream == null) { return 0; } return booleanOutputStream.getBufferedBytes(); }
BooleanOutputStream output = new BooleanOutputStream(buffer); output.writeBooleans(count, true); output.recordCheckpoint(); output.close(); List<BooleanStreamCheckpoint> batchWriteCheckpoints = output.getCheckpoints(); DynamicSliceOutput slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice); output.reset(); for (int count : counts) { for (int i = 0; i < count; i++) { output.writeBoolean(true); output.recordCheckpoint(); output.close(); List<BooleanStreamCheckpoint> singleWriteCheckpoints = output.getCheckpoints(); slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice);
outputStream.reset(); outputStream.writeBooleans(32, true); expectedValues.addAll(Collections.nCopies(32, true)); outputStream.writeBooleans(32, false); expectedValues.addAll(Collections.nCopies(32, false)); outputStream.writeBooleans(1, true); expectedValues.add(true); outputStream.writeBooleans(1, false); expectedValues.add(false); outputStream.writeBooleans(34, true); expectedValues.addAll(Collections.nCopies(34, true)); outputStream.writeBooleans(34, false); expectedValues.addAll(Collections.nCopies(34, false)); outputStream.writeBoolean(true); expectedValues.add(true); outputStream.writeBoolean(false); expectedValues.add(false); outputStream.close(); StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(33); streamDataOutput.writeData(sliceOutput); Stream stream = streamDataOutput.getStream();
@Override protected void writeValue(BooleanOutputStream outputStream, Boolean value) { outputStream.writeBoolean(value); }
public Optional<StreamDataOutput> getStreamDataOutput(int column) { checkArgument(closed); if (booleanOutputStream == null) { return Optional.empty(); } StreamDataOutput streamDataOutput = booleanOutputStream.getStreamDataOutput(column); // rewrite the DATA stream created by the boolean output stream to a PRESENT stream Stream stream = new Stream(column, PRESENT, toIntExact(streamDataOutput.size()), streamDataOutput.getStream().isUseVInts()); return Optional.of(new StreamDataOutput( sliceOutput -> { streamDataOutput.writeData(sliceOutput); return stream.getLength(); }, stream)); }
public BooleanColumnWriter(int column, Type type, CompressionKind compression, int bufferSize) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.dataStream = new BooleanOutputStream(compression, bufferSize); this.presentStream = new PresentOutputStream(compression, bufferSize); }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); }
@Override public void reset() { closed = false; dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); statisticsBuilder = new BooleanStatisticsBuilder(); } }
@Override public void close() { closed = true; if (bitsInData > 0) { flushData(); } byteOutputStream.close(); }