private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "Writing columns with non-zero sequence IDs is not supported in ORC: " + columnEncodings); return OrcProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
private static DwrfProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "DWRF writer doesn't support writing columns with non-zero sequence IDs: " + columnEncodings); return DwrfProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
@Override public void startStripe(InputStreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) { dictionaryDataStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, DICTIONARY_DATA, LongInputStream.class); dictionarySize = encoding.get(streamDescriptor.getStreamId()) .getColumnEncoding(streamDescriptor.getSequence()) .getDictionarySize(); dictionaryOpen = false; inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); presentStreamSource = missingStreamSource(BooleanInputStream.class); dataStreamSource = missingStreamSource(LongInputStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }
@Override public void startStripe(InputStreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) { stripeDictionaryDataStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, DICTIONARY_DATA, ByteArrayInputStream.class); stripeDictionaryLengthStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); stripeDictionarySize = encoding.get(streamDescriptor.getStreamId()) .getColumnEncoding(streamDescriptor.getSequence()) .getDictionarySize(); stripeDictionaryOpen = false; presentStreamSource = missingStreamSource(BooleanInputStream.class); dataStreamSource = missingStreamSource(LongInputStream.class); inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); rowGroupDictionaryLengthStreamSource = missingStreamSource(RowGroupDictionaryLengthInputStream.class); rowGroupDictionaryDataStreamSource = missingStreamSource(ByteArrayInputStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }
private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "Writing columns with non-zero sequence IDs is not supported in ORC: " + columnEncodings); return OrcProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
private static DwrfProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "DWRF writer doesn't support writing columns with non-zero sequence IDs: " + columnEncodings); return DwrfProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
@Override public void startStripe(StreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) throws IOException { dictionaryDataStreamSource = dictionaryStreamSources.getStreamSource(streamDescriptor, DICTIONARY_DATA, LongStream.class); dictionarySize = encoding.get(streamDescriptor.getStreamId()).getDictionarySize(); dictionaryOpen = false; inDictionaryStreamSource = missingStreamSource(BooleanStream.class); presentStreamSource = missingStreamSource(BooleanStream.class); dataStreamSource = missingStreamSource(LongStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }
@Override public void startStripe(InputStreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) { dictionaryDataStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, DICTIONARY_DATA, LongInputStream.class); dictionarySize = encoding.get(streamDescriptor.getStreamId()) .getColumnEncoding(streamDescriptor.getSequence()) .getDictionarySize(); dictionaryOpen = false; inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); presentStreamSource = missingStreamSource(BooleanInputStream.class); dataStreamSource = missingStreamSource(LongInputStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }
@Override public void startStripe(StreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) throws IOException { dictionaryDataStreamSource = dictionaryStreamSources.getStreamSource(streamDescriptor, DICTIONARY_DATA, ByteArrayStream.class); dictionaryLengthStreamSource = dictionaryStreamSources.getStreamSource(streamDescriptor, LENGTH, LongStream.class); dictionarySize = encoding.get(streamDescriptor.getStreamId()).getDictionarySize(); dictionaryOpen = false; presentStreamSource = missingStreamSource(BooleanStream.class); dataStreamSource = missingStreamSource(LongStream.class); inDictionaryStreamSource = missingStreamSource(BooleanStream.class); rowGroupDictionaryLengthStreamSource = missingStreamSource(RowGroupDictionaryLengthStream.class); rowGroupDictionaryDataStreamSource = missingStreamSource(ByteArrayStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }
@Override public void startStripe(InputStreamSources dictionaryStreamSources, List<ColumnEncoding> encoding) { stripeDictionaryDataStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, DICTIONARY_DATA, ByteArrayInputStream.class); stripeDictionaryLengthStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); stripeDictionarySize = encoding.get(streamDescriptor.getStreamId()) .getColumnEncoding(streamDescriptor.getSequence()) .getDictionarySize(); stripeDictionaryOpen = false; presentStreamSource = missingStreamSource(BooleanInputStream.class); dataStreamSource = missingStreamSource(LongInputStream.class); inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); rowGroupDictionaryLengthStreamSource = missingStreamSource(RowGroupDictionaryLengthInputStream.class); rowGroupDictionaryDataStreamSource = missingStreamSource(ByteArrayInputStream.class); readOffset = 0; nextBatchSize = 0; presentStream = null; inDictionaryStream = null; dataStream = null; rowGroupOpen = false; }