@Override public void setInitialCapacity(int numRecords) { for (final ValueVector v : (Iterable<ValueVector>) this) { v.setInitialCapacity(numRecords); } }
@Override public void setInitialCapacity(int valueCount, double density) { for (final ValueVector vector : (Iterable<ValueVector>) this) { if (vector instanceof DensityAwareVector) { ((DensityAwareVector)vector).setInitialCapacity(valueCount, density); } else { vector.setInitialCapacity(valueCount); } } }
public void setInitialCapacity(int initialCapacity) { for (VectorWrapper<?> w : wrappers) { final ValueVector vv = w.getValueVector(); vv.setInitialCapacity(initialCapacity); } }
OutgoingBatch(int batchIdx, int nextBatchIdx, int maxRecords, final VectorAccessible incoming, BufferAllocator allocator, AccountingExecTunnel tunnel, HashPartitionSender config, OperatorContext context, int oppositeMinorFragmentId, OperatorStats stats) { Preconditions.checkArgument(maxRecords <= Character.MAX_VALUE, "maxRecords cannot exceed " + Character.MAX_VALUE); this.batchIdx = batchIdx; this.nextBatchIdx = nextBatchIdx; this.maxRecords = maxRecords; this.tunnel = tunnel; this.config = config; this.context = context; this.oppositeMinorFragmentId = oppositeMinorFragmentId; this.stats = stats; for (VectorWrapper<?> v : incoming) { ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator); outgoingVector.setInitialCapacity(maxRecords); add(outgoingVector); if (outgoingVector instanceof VarBinaryVector) { varbins.add(((VarBinaryVector) outgoingVector)); } else if (outgoingVector instanceof VarCharVector) { varchars.add(((VarCharVector) outgoingVector)); } } }
private void allocateVectors(int targetRecordCount) { boolean memoryAllocated = false; double density = lastSuccessfulDensity; while (!memoryAllocated) { try { for (VectorWrapper<?> w : outgoing) { final ValueVector v = w.getValueVector(); if (v instanceof DensityAwareVector) { ((DensityAwareVector) v).setInitialCapacity(targetRecordCount, density); } else { v.setInitialCapacity(targetRecordCount); } v.allocateNew(); } memoryAllocated = true; lastSuccessfulDensity = density; } catch (OutOfMemoryException ex) { // halve the density and try again density = density / 2; if (density < 0.01) { logger.debug("PriorityQueueCopierTemplate ran out of memory to allocate outgoing batch. " + "Records: {}, density: {}", targetRecordCount, density); throw ex; } // try allocating again with lower density logger.debug("PriorityQueueCopierTemplate: Ran out of memory. Retrying allocation with lower density."); } } }
@Override public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException { int estimatedRecordCount; if ((reader != null) && (reader.getInput() != null) && (vectorMap.size() > 0)) { final OptionManager options = context.getOptions(); final int listSizeEstimate = (int) options.getOption(ExecConstants.BATCH_LIST_SIZE_ESTIMATE); final int varFieldSizeEstimate = (int) options.getOption(ExecConstants.BATCH_VARIABLE_FIELD_SIZE_ESTIMATE); final int estimatedRecordSize = BatchSchema.estimateRecordSize(vectorMap, listSizeEstimate, varFieldSizeEstimate); if (estimatedRecordSize > 0) { estimatedRecordCount = (int) Math.min(reader.getInput().length / estimatedRecordSize, numRowsPerBatch); } else { estimatedRecordCount = (int) numRowsPerBatch; } } else { estimatedRecordCount = (int) numRowsPerBatch; } for (final ValueVector v : vectorMap.values()) { v.setInitialCapacity(estimatedRecordCount); v.allocateNew(); } }
/** * Initialize the OutgoingBatch based on the current schema in incoming RecordBatch */ public void initializeBatch() { for (VectorWrapper<?> v : incoming) { // create new vector ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator); outgoingVector.setInitialCapacity(maxRecordCount); vectorContainer.add(outgoingVector); } vectorContainer.allocateNew(); doSetup(incoming, vectorContainer); }
@Override public DateDayWriter dateDay(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DateDayVector v = container.addOrGet(name, FieldType.nullable( MinorType.DATEDAY.getType() ), DateDayVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DATEDAY); } } return writer; }
@Override public TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampNanoTZVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, timezone) ), TimeStampNanoTZVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANOTZ); } } return writer; }
@Override public DecimalWriter decimal(String name, int scale, int precision) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DecimalVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale) ), DecimalVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DECIMAL); } } return writer; }
@Override public UInt1Writer uInt1(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt1Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT1.getType() ), UInt1Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT1); } } return writer; }
@Override public UInt4Writer uInt4(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt4Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT4.getType() ), UInt4Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT4); } } return writer; }
@Override public DateDayWriter dateDay(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DateDayVector v = container.addOrGet(name, FieldType.nullable( MinorType.DATEDAY.getType() ), DateDayVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DATEDAY); } } return writer; }
@Override public IntervalYearWriter intervalYear(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); IntervalYearVector v = container.addOrGet(name, FieldType.nullable( MinorType.INTERVALYEAR.getType() ), IntervalYearVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.INTERVALYEAR); } } return writer; }
@Override public UInt8Writer uInt8(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt8Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT8.getType() ), UInt8Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT8); } } return writer; }
@Override public TimeStampNanoWriter timeStampNano(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampNanoVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMESTAMPNANO.getType() ), TimeStampNanoVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANO); } } return writer; }
@Override public TimeStampMicroTZWriter timeStampMicroTZ(String name, String timezone) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampMicroTZVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, timezone) ), TimeStampMicroTZVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPMICROTZ); } } return writer; }
@Override public Float8Writer float8(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); Float8Vector v = container.addOrGet(name, FieldType.nullable( MinorType.FLOAT8.getType() ), Float8Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.FLOAT8); } } return writer; }
@Override public TimeMicroWriter timeMicro(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeMicroVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMEMICRO.getType() ), TimeMicroVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMEMICRO); } } return writer; }
@Override public DecimalWriter decimal(String name, int scale, int precision) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DecimalVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale) ), DecimalVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DECIMAL); } } return writer; }