public VectorizedRowBatch createRowBatch(boolean useDecimal64) { return useDecimal64 ? this.schema.createRowBatchV2() : this.schema.createRowBatch(); }
protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options) throws IOException { super(fileReader, options); batch = this.schema.createRowBatch(); rowInBatch = 0; }
public VectorizedRowBatch createRowBatch() { return this.schema.createRowBatch(); }
protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options, final Configuration conf) throws IOException { super(fileReader, options); final boolean useDecimal64ColumnVectors = conf != null && HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); if (useDecimal64ColumnVectors){ batch = this.schema.createRowBatchV2(); } else { batch = this.schema.createRowBatch(); } rowInBatch = 0; }
public DeleteReaderValue(Reader deleteDeltaReader, Reader.Options readerOptions, int bucket, ValidTxnList validTxnList) throws IOException { this.recordReader = deleteDeltaReader.rowsOptions(readerOptions); this.bucketForSplit = bucket; this.batch = deleteDeltaReader.getSchema().createRowBatch(); if (!recordReader.nextBatch(batch)) { // Read the first batch. this.batch = null; // Oh! the first batch itself was null. Close the reader. } this.indexPtrInBatch = 0; this.validTxnList = validTxnList; }
DeleteReaderValue(Reader deleteDeltaReader, Path deleteDeltaFile, Reader.Options readerOptions, int bucket, ValidWriteIdList validWriteIdList, boolean isBucketedTable, final JobConf conf, OrcRawRecordMerger.KeyInterval keyInterval, OrcSplit orcSplit) throws IOException { this.reader = deleteDeltaReader; this.deleteDeltaFile = deleteDeltaFile; this.recordReader = deleteDeltaReader.rowsOptions(readerOptions, conf); this.bucketForSplit = bucket; final boolean useDecimal64ColumnVector = HiveConf.getVar(conf, ConfVars .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); if (useDecimal64ColumnVector) { this.batch = deleteDeltaReader.getSchema().createRowBatchV2(); } else { this.batch = deleteDeltaReader.getSchema().createRowBatch(); } if (!recordReader.nextBatch(batch)) { // Read the first batch. this.batch = null; // Oh! the first batch itself was null. Close the reader. } this.indexPtrInBatch = 0; this.validWriteIdList = validWriteIdList; this.isBucketedTable = isBucketedTable; if(batch != null) { checkBucketId();//check 1st batch } this.keyInterval = keyInterval; this.orcSplit = orcSplit; this.numEvents = deleteDeltaReader.getNumberOfRows(); LOG.debug("Num events stats({},x,x)", numEvents); }
WriterImpl(FileSystem fs, Path path, OrcFile.WriterOptions opts) throws IOException { super(fs, path, opts); this.inspector = opts.getInspector(); this.internalBatch = opts.getSchema().createRowBatch(opts.getBatchSize()); this.fields = initializeFieldsFromOi(inspector); }
.setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE)); VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000); batch.size = 1000; LongColumnVector lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]);
WriterImpl(FileSystem fs, Path path, OrcFile.WriterOptions opts) throws IOException { super(fs, path, opts); this.inspector = opts.getInspector(); boolean useDecimal64ColumnVectors = opts.getConfiguration() != null && HiveConf.getVar(opts.getConfiguration(), HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) .equalsIgnoreCase("decimal_64"); if (useDecimal64ColumnVectors) { this.internalBatch = opts.getSchema().createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64, opts.getBatchSize()); } else { this.internalBatch = opts.getSchema().createRowBatch(opts.getBatchSize()); } this.fields = initializeFieldsFromOi(inspector); }
public JsonORCFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException { Configuration conf = new Configuration(); Path path = new Path(logFilePath.getLogFilePath()); schema = schemaProvider.getSchema(logFilePath.getTopic(), logFilePath); List<TypeDescription> fieldTypes = schema.getChildren(); converters = new JsonConverter[fieldTypes.size()]; for (int c = 0; c < converters.length; ++c) { converters[c] = VectorColumnFiller.createConverter(fieldTypes .get(c)); } writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf) .compress(resolveCompression(codec)).setSchema(schema)); batch = schema.createRowBatch(); }
@SuppressWarnings("deprecation") public JsonORCFileReader(LogFilePath logFilePath, CompressionCodec codec) throws IOException { schema = schemaProvider.getSchema(logFilePath.getTopic(), logFilePath); Path path = new Path(logFilePath.getLogFilePath()); Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(new Configuration(true))); offset = logFilePath.getOffset(); rows = reader.rows(); batch = reader.getSchema().createRowBatch(); rows.nextBatch(batch); }
this.rowBatch = schema.createRowBatch(batchSize); rowsInBatch = 0; nextRow = 0;
.setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE)); VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000); batch.size = 1000; StructColumnVector scv = (StructColumnVector)batch.cols[5];
VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000); batch.size = 1000; StructColumnVector scv = (StructColumnVector)batch.cols[5];
.setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE)); VectorizedRowBatch batch = fileSchema.createRowBatch(1000); batch.size = 1000; LongColumnVector lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]); RecordReader rows = reader.rowsOptions(new Reader.Options() .schema(readerSchema)); batch = readerSchema.createRowBatch(); lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]); LongColumnVector future1 = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[1]); .schema(readerSchema) .include(new boolean[]{false, true, true, true, false, false, true})); batch = readerSchema.createRowBatch(); lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]); future1 = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[1]);
OrcIterator(Path filename, TypeDescription schema, RecordReader rows) { this.filename = filename; this.rows = rows; this.batch = schema.createRowBatch(); }
public OrcMapreduceRecordWriter(Writer writer) { this.writer = writer; schema = writer.getSchema(); this.batch = schema.createRowBatch(); isTopStruct = schema.getCategory() == TypeDescription.Category.STRUCT; }
public SparkOrcWriter(OrcFileAppender writer) { TypeDescription schema = writer.getSchema(); batch = schema.createRowBatch(BATCH_SIZE); this.writer = writer; converters = buildConverters(schema); }
StructType partitionSchema, InternalRow partitionValues) { batch = orcSchema.createRowBatch(capacity); assert(!batch.selectedInUse); // `selectedInUse` should be initialized with `false`.
StructType partitionSchema, InternalRow partitionValues) { batch = orcSchema.createRowBatch(capacity); assert(!batch.selectedInUse); // `selectedInUse` should be initialized with `false`.