@Override public List<FieldVector> getChildren() { return Lists.newArrayList(Iterables.transform(mutator.getVectors(), new Function<ValueVector, FieldVector>() { @Nullable @Override public FieldVector apply(@Nullable ValueVector input) { return (FieldVector) input; } })); }
@Override public <T extends FieldVector> T addOrGet(String childName, FieldType fieldType, Class<T> clazz) { try { Field field = new Field(childName, fieldType, null); final FieldVector v = mutator.addField(field, clazz); putChild(childName, v); return this.typeify(v, clazz); } catch (SchemaChangeException e) { throw new IllegalStateException(e); } }
@Override public void setup(OutputMutator output) throws ExecutionSetupException { this.externalCallback = output.getCallBack(); for (ValueVector v : output.getVectors()) { final Field f = v.getField(); mutator.addField(f, (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(f)); copierToOutputTransfers.add(wrapper.getValueVector().makeTransferPair(output.getVector(field.getName())));
@Override public <T extends FieldVector> T getChild(String name, Class<T> clazz) { final ValueVector v = mutator.getVector(name.toLowerCase()); if (v == null) { return null; } return typeify(v, clazz); } }
w = new DoubleWriter(f); } else if(type.isEnum()) { w = new EnumWriter(f, output.getManagedBuffer()); } else if(type == String.class) { w = new StringWriter(f, output.getManagedBuffer()); } else if (type == Timestamp.class) { w = new TimeStampMilliWriter(f);
public VectorContainerWriter(OutputMutator mutator) { this.mutator = mutator; this.allocator = new RootAllocator(Long.MAX_VALUE); structVector = new SpecialStructVector(allocator, mutator.getCallBack()); structRoot = new SingleStructWriter(structVector); }
mutator.isSchemaChanged(); } else { if (mutator.isSchemaChanged()) {
public void materializeVectors(List<SchemaPath> columns, OutputMutator mutator) { Preconditions.checkNotNull(columns, "A scan's column selection cannot be null."); Set<String> selectedColumns = new HashSet<>(); for (SchemaPath sp : columns) { selectedColumns.add(sp.getRootSegment().getNameSegment().getPath()); } for (Field field : this) { if (columns != null && !selectedColumns.contains("*") && !selectedColumns.contains(field.getName())) { continue; } mutator.addField(field, CompleteType.fromField(field).getValueVectorClass()); } }
@Override public void setup(OutputMutator output) throws ExecutionSetupException { computeLocality(footer); splitColumns(footer, vectorizableReaderColumns, nonVectorizableReaderColumns); final ExecutionPath execPath = getExecutionPath(); delegates = execPath.getReaders(this); Preconditions.checkArgument(!delegates.isEmpty(), "There should be at least one delegated RecordReader"); for (RecordReader delegateReader : delegates) { delegateReader.setup(output); } for (SchemaPath path : vectorizableReaderColumns) { String name = path.getRootSegment().getNameSegment().getPath(); vectorizedMap.put(name, output.getVector(name)); } for (SchemaPath path : nonVectorizableReaderColumns) { String name = path.getRootSegment().getNameSegment().getPath(); nonVectorizedMap.put(name, output.getVector(name)); } context.getStats().setLongStat(Metric.PARQUET_EXEC_PATH, execPath.ordinal()); context.getStats().setLongStat(Metric.NUM_VECTORIZED_COLUMNS, vectorizableReaderColumns.size()); context.getStats().setLongStat(Metric.NUM_NON_VECTORIZED_COLUMNS, nonVectorizableReaderColumns.size()); context.getStats().setLongStat(Metric.FILTER_EXISTS, filterConditions != null && filterConditions.size() > 0 ? 1 : 0); }
rootWriter = new VectorContainerWriter(outputMutator); listWriter = rootWriter.rootAsStruct().list(COL_NAME); tmpBuf = outputMutator.getManagedBuffer();
if (selectedFields[i]) { Field field = new Field(outputColumns.get(i), true, MinorType.VARCHAR.getType(), null); this.vectors[i] = outputMutator.addField(field, VarCharVector.class);
public void newSchema() { incoming.buildSchema(); for (Field field : targetSchema.getFields()) { ValueVector vector = outputMutator.getVector(field.getName()); if (vector == null) { continue;
DecimalWriter writer = isRepeated ? list(name).decimal() : getWriterProvider().decimal(name, type.getDecimalMetadata().getScale(), type.getDecimalMetadata().getPrecision()); return new Decimal9Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale(), mutator.getManagedBuffer()); DecimalWriter writer = isRepeated ? list(name).decimal() : getWriterProvider().decimal(name, metadata.getScale(), metadata.getPrecision()); return new Decimal18Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale(), mutator.getManagedBuffer()); } else { VarBinaryWriter writer = type.getRepetition() == Repetition.REPEATED ? getWriterProvider().list(name).varBinary() : getWriterProvider().varBinary(name); return new FixedBinaryToVarbinaryConverter(writer, DeprecatedParquetVectorizedReader.getTypeLengthInBits(type.getPrimitiveTypeName()) / 8, mutator.getManagedBuffer()); case UTF8: { VarCharWriter writer = isRepeated ? list(name).varChar() : getWriterProvider().varChar(name); return new VarCharConverter(writer, mutator.getManagedBuffer()); DecimalMetadata metadata = type.getDecimalMetadata(); DecimalWriter writer = isRepeated ? list(name).decimal() : getWriterProvider().decimal(name, metadata.getScale(), metadata.getPrecision()); return new BinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); return new VarCharConverter(writer, mutator.getManagedBuffer()); return new VarBinaryConverter(writer, mutator.getManagedBuffer()); DecimalMetadata metadata = type.getDecimalMetadata(); DecimalWriter writer = isRepeated ? list(name).decimal() : getWriterProvider().decimal(name, metadata.getScale(), metadata.getPrecision()); return new BinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); return new VarCharConverter(writer, mutator.getManagedBuffer());
@Override public void setup(OutputMutator output) throws ExecutionSetupException { final SequenceFileAsBinaryInputFormat inputFormat = new SequenceFileAsBinaryInputFormat(); final JobConf jobConf = new JobConf(dfs.getConf()); jobConf.setInputFormat(inputFormat.getClass()); reader = getRecordReader(inputFormat, jobConf); final Field keyField = new Field(keySchema, true, getArrowTypeForMajorType(KEY_TYPE), null); final Field valueField = new Field(valueSchema, true, getArrowTypeForMajorType(VALUE_TYPE), null); try { keyVector = output.addField(keyField, VarBinaryVector.class); valueVector = output.addField(valueField, VarBinaryVector.class); } catch (SchemaChangeException sce) { throw new ExecutionSetupException("Error in setting up sequencefile reader.", sce); } }
@Override public void init(OutputMutator output) throws SchemaChangeException { // MaterializedField mf = MajorTypeHelper.getFieldForNameAndMajorType(field.getName(), type); org.apache.arrow.vector.types.pojo.Field mf = new org.apache.arrow.vector.types.pojo.Field(field.getName(), true, getArrowMinorType(type.getMinorType()).getType(), null); @SuppressWarnings("unchecked") Class<V> valueVectorClass = (Class<V>) TypeHelper.getValueVectorClass(getArrowMinorType(type.getMinorType())); this.vector = output.addField(mf, valueVectorClass); }