@Override public void finishBatch() { for (int i = 0; i <= maxField; i++) { if (this.vectors[i] != null) { this.vectors[i].setValueCount(recordCount); } } }
public void populate(final int count){ for (int i = 0; i < count; i++) { if(value != null) { ((VarCharVector) vector).setSafe(i, value, 0, value.length); } } vector.setValueCount(count); }
private static void writeVarCharData(VarCharVector vector, int valueCount) { Charset utf8Charset = Charset.forName("UTF-8"); final byte[] even = "AAAAA1".getBytes(utf8Charset); final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); for (int i = 0; i < valueCount; i++) { if (i % 2 == 0) { vector.setSafe(i, even); } else { vector.setSafe(i, odd); } } vector.setValueCount(valueCount); } }
/** * * @param allocator * @param name * @param length the length of the vector * @return */ public static VarCharVector stringVectorOf(BufferAllocator allocator,String name,int length) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); float4Vector.setValueCount(length); return float4Vector; }
/** * * @param allocator * @param name * @param data * @return */ public static VarCharVector vectorFor(BufferAllocator allocator,String name,String[] data) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); for(int i = 0; i < data.length; i++) { float4Vector.setSafe(i,data[i].getBytes()); } float4Vector.setValueCount(data.length); return float4Vector; }
public void writeNull() { vector.setNull(idx()); vector.setValueCount(idx()+1); } }
public void writeVarChar(int start, int end, ArrowBuf buffer) { vector.setSafe(idx(), 1, start, end, buffer); vector.setValueCount(idx()+1); }
public void writeVarChar(Text value) { vector.setSafe(idx(), value); vector.setValueCount(idx()+1); }
public void write(VarCharHolder h) { vector.setSafe(idx(), h); vector.setValueCount(idx()+1); }
public void write(NullableVarCharHolder h) { vector.setSafe(idx(), h); vector.setValueCount(idx()+1); }
static String[] populateVarCharValues(VarCharVector vector, int size){ vector.allocateNew(); String values[] = new String[size]; for(int i =0; i < values.length; i++){ if (RAND.nextBoolean()) { values[i] = RandomStringUtils.randomAlphanumeric(RAND.nextInt(25)); vector.setSafe(i, values[i].getBytes(Charsets.UTF_8)); } } vector.setValueCount(values.length); return values; }
static String[] populateVarCharValuesWithoutNull(VarCharVector vector, int size){ vector.allocateNew(); String values[] = new String[size]; for(int i =0; i < values.length; i++){ values[i] = RandomStringUtils.randomAlphanumeric(RAND.nextInt(25)); vector.setSafe(i, values[i].getBytes(Charsets.UTF_8)); } vector.setValueCount(values.length); return values; }
static void populate(VarCharVector vector, byte[][] values){ vector.allocateNew(); Random r = new Random(); for(int i =0; i < values.length; i++){ byte[] val = values[i]; if(val != null){ vector.setSafe(i, val, 0, val.length); } else { // add noise. this confirms that after pivot, noise is gone. byte[] bytes = new byte[r.nextInt(15)]; r.nextBytes(bytes); vector.setSafe(i, bytes, 0, bytes.length); vector.setNull(i); } } vector.setValueCount(values.length); }
final VarBinaryVector dictionary = dictionaries.get(fieldName).getValueAccessorById(VarBinaryVector.class, 0).getValueVector(); decodeVarChar(input, output, dictionary); output.setValueCount(recordsConsumedCurrentBatch);
compareArray[i] = s; varCharVector.setValueCount(valueCount); final int length = startLength[1]; tp.splitAndTransfer(start, length); newVarCharVector.setValueCount(length); for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0;
in.setValueCount(count); copy(copiers, sv2); out.setValueCount(count); for(int i =0; i < count; i++){ assertEquals(in.getObject(i), out.getObject(i));
@Test public void varcharRoundtrip() throws Exception { final int count = 1024; try ( VarCharVector in = new VarCharVector("in", allocator); VarCharVector out = new VarCharVector("out", allocator); ) { in.allocateNew(count * 8, count); for (int i = 0; i < count; i++) { if (i % 5 == 0) { byte[] data = ("hello-" + i).getBytes(Charsets.UTF_8); in.setSafe(i, data, 0, data.length); } } in.setValueCount(count); final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out)); try ( final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth()); final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount()); ) { fbv.ensureAvailableBlocks(count); Pivots.pivot(pivot, count, fbv, vbv); ValueVector[] ins = new ValueVector[]{in}; ValueVector[] outs = new ValueVector[]{out}; unpivotHelper(pivot, fbv, vbv, ins, outs, 0, count); unpivotHelper(pivot, fbv, vbv, ins, outs, 0, 100); unpivotHelper(pivot, fbv, vbv, ins, outs, 100, 924); } } }