/** * Set the maximum number of rows in the batch. * Data is not preserved. */ public void ensureSize(int rows) { for (ColumnVector col : cols) { col.ensureSize(rows, false); } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); for(int i=0; i < fields.length; ++i) { fields[i].ensureSize(size, preserveData); } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (tags.length < size) { if (preserveData) { int[] oldTags = tags; tags = new int[size]; System.arraycopy(oldTags, 0, tags, 0, oldTags.length); } else { tags = new int[size]; } for(int i=0; i < fields.length; ++i) { fields[i].ensureSize(size, preserveData); } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size > vector.length) { double[] oldArray = vector; vector = new double[size]; if (preserveData) { if (isRepeating) { vector[0] = oldArray[0]; } else { System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); } } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size > vector.length) { long[] oldArray = vector; vector = new long[size]; if (preserveData) { if (isRepeating) { vector[0] = oldArray[0]; } else { System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); } } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size <= totalSeconds.length) return; long[] oldTime = totalSeconds; int[] oldNanos = nanos; totalSeconds = new long[size]; nanos = new int[size]; if (preserveData) { if (isRepeating) { totalSeconds[0] = oldTime[0]; nanos[0] = oldNanos[0]; } else { System.arraycopy(oldTime, 0, totalSeconds, 0, oldTime.length); System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size > offsets.length) { long[] oldOffsets = offsets; offsets = new long[size]; long oldLengths[] = lengths; lengths = new long[size]; if (preserveData) { if (isRepeating) { offsets[0] = oldOffsets[0]; lengths[0] = oldLengths[0]; } else { System.arraycopy(oldOffsets, 0, offsets, 0 , oldOffsets.length); System.arraycopy(oldLengths, 0, lengths, 0, oldLengths.length); } } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size <= time.length) return; long[] oldTime = time; int[] oldNanos = nanos; time = new long[size]; nanos = new int[size]; if (preserveData) { if (isRepeating) { time[0] = oldTime[0]; nanos[0] = oldNanos[0]; } else { System.arraycopy(oldTime, 0, time, 0, oldTime.length); System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size > vector.length) { int[] oldStart = start; start = new int[size]; int[] oldLength = length; length = new int[size]; byte[][] oldVector = vector; vector = new byte[size][]; if (preserveData) { if (isRepeating) { vector[0] = oldVector[0]; start[0] = oldStart[0]; length[0] = oldLength[0]; } else { System.arraycopy(oldVector, 0, vector, 0, oldVector.length); System.arraycopy(oldStart, 0, start, 0 , oldStart.length); System.arraycopy(oldLength, 0, length, 0, oldLength.length); } } } }
@Override public void ensureSize(int size, boolean preserveData) { super.ensureSize(size, preserveData); if (size <= vector.length) return; // We assume the existing vector is always valid. HiveDecimalWritable[] oldArray = vector; vector = new HiveDecimalWritable[size]; int initPos = 0; if (preserveData) { // we copy all of the values to avoid creating more objects // TODO: it might be cheaper to always preserve data or reset existing objects initPos = oldArray.length; System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); } for (int i = initPos; i < vector.length; ++i) { vector[i] = new HiveDecimalWritable(0); // Initially zero. } }
public void convert(JsonElement value, ColumnVector vect, int row) { if (value == null || value.isJsonNull()) { vect.noNulls = false; vect.isNull[row] = true; } else { ListColumnVector vector = (ListColumnVector) vect; JsonArray obj = value.getAsJsonArray(); vector.lengths[row] = obj.size(); vector.offsets[row] = vector.childCount; vector.childCount += vector.lengths[row]; vector.child.ensureSize(vector.childCount, true); for (int c = 0; c < obj.size(); ++c) { childrenConverter.convert(obj.get(c), vector.child, (int) vector.offsets[row] + c); } } } }
childCount += length; lengths[outputElementNum] = length; child.ensureSize(childCount, true); for (int i = 0; i < length; ++i) { final int outputIndex = i + offset;
childCount += length; lengths[outputElementNum] = length; keys.ensureSize(childCount, true); values.ensureSize(childCount, true); for (int i = 0; i < length; ++i) { final int inputIndex = inputOffset + i;
private void storeMapRowColumn(ColumnVector colVector, Field field, int batchIndex, boolean canRetainByteRef) throws IOException { final MapColumnVector mapColVector = (MapColumnVector) colVector; final MapComplexTypeHelper mapHelper = (MapComplexTypeHelper) field.getComplexHelper(); final ColumnVector keysColVector = mapColVector.keys; final ColumnVector valuesColVector = mapColVector.values; int offset = mapColVector.childCount; mapColVector.offsets[batchIndex] = offset; mapColVector.isNull[batchIndex] = false; int keyValueCount = 0; while (deserializeRead.isNextComplexMultiValue()) { // Ensure child size. final int childCapacity = mapColVector.keys.isNull.length; final int childCount = mapColVector.childCount; if (childCapacity < childCount / 0.75) { mapColVector.keys.ensureSize(childCapacity * 2, true); mapColVector.values.ensureSize(childCapacity * 2, true); } // Key. storeComplexFieldRowColumn( keysColVector, mapHelper.getKeyField(), offset, canRetainByteRef); // Value. storeComplexFieldRowColumn( valuesColVector, mapHelper.getValueField(), offset, canRetainByteRef); offset++; keyValueCount++; } mapColVector.childCount += keyValueCount; mapColVector.lengths[batchIndex] = keyValueCount; }
private void storeListRowColumn(ColumnVector colVector, Field field, int batchIndex, boolean canRetainByteRef) throws IOException { final ListColumnVector listColVector = (ListColumnVector) colVector; final ColumnVector elementColVector = listColVector.child; int offset = listColVector.childCount; listColVector.isNull[batchIndex] = false; listColVector.offsets[batchIndex] = offset; final ListComplexTypeHelper listHelper = (ListComplexTypeHelper) field.getComplexHelper(); int listLength = 0; while (deserializeRead.isNextComplexMultiValue()) { // Ensure child size. final int childCapacity = listColVector.child.isNull.length; final int childCount = listColVector.childCount; if (childCapacity < childCount / 0.75) { listColVector.child.ensureSize(childCapacity * 2, true); } storeComplexFieldRowColumn( elementColVector, listHelper.getElementField(), offset, canRetainByteRef); offset++; listLength++; } listColVector.childCount += listLength; listColVector.lengths[batchIndex] = listLength; }
listColumnVector.lengths[batchIndex] = size; listColumnVector.childCount = childCount + size; listColumnVector.child.ensureSize(childCount + size, true); mapColumnVector.offsets[batchIndex] = childCount; mapColumnVector.lengths[batchIndex] = size; mapColumnVector.keys.ensureSize(childCount + size, true); mapColumnVector.values.ensureSize(childCount + size, true);
vector.offsets[rowId] = offset; vector.lengths[rowId] = length; vector.child.ensureSize(offset + length, true); vector.childCount += length; for (int c = 0; c < length; ++c) { vector.offsets[rowId] = offset; vector.lengths[rowId] = length; vector.keys.ensureSize(offset + length, true); vector.values.ensureSize(offset + length, true); vector.childCount += length; for (Object item: map) {
vector.offsets[rowId] = offset; vector.lengths[rowId] = length; vector.child.ensureSize(offset + length, true); vector.childCount += length; for (int c = 0; c < length; ++c) { vector.offsets[rowId] = offset; vector.lengths[rowId] = length; vector.keys.ensureSize(offset + length, true); vector.values.ensureSize(offset + length, true); vector.childCount += length; for (Object item: map) {
cv.ensureSize(batchSize, false); reader.nextVector(cv, null, batchSize);
/** * Called at the top level to read into the given batch. * @param batch the batch to read into * @param batchSize the number of rows to read * @throws IOException */ public void nextBatch(VectorizedRowBatch batch, int batchSize) throws IOException { batch.cols[0].reset(); batch.cols[0].ensureSize(batchSize, false); nextVector(batch.cols[0], null, batchSize); }