@Override public void allocate() { vector.allocateNew(); }
public void allocate(){ vector.allocateNew(); }
/** * * @param allocator * @param name * @param length the length of the vector * @return */ public static VarCharVector stringVectorOf(BufferAllocator allocator,String name,int length) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); float4Vector.setValueCount(length); return float4Vector; }
/** * * @param allocator * @param name * @param data * @return */ public static VarCharVector vectorFor(BufferAllocator allocator,String name,String[] data) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); for(int i = 0; i < data.length; i++) { float4Vector.setSafe(i,data[i].getBytes()); } float4Vector.setValueCount(data.length); return float4Vector; }
/** Helper method which creates a test varchar vector */ private static VarCharVector testVarCharVector() { VarCharVector colVarCharV = new VarCharVector("colVarChar", ALLOCATOR); colVarCharV.allocateNew(500, 5); for(int i=0; i<TEST_VARCHAR_VALUES.size(); i++) { if (TEST_VARCHAR_VALUES.get(i) == null) { colVarCharV.setNull(i); } else { colVarCharV.set(i, TEST_VARCHAR_VALUES.get(i).getBytes()); } } return colVarCharV; }
static String[] populateVarCharValues(VarCharVector vector, int size){ vector.allocateNew(); String values[] = new String[size]; for(int i =0; i < values.length; i++){ if (RAND.nextBoolean()) { values[i] = RandomStringUtils.randomAlphanumeric(RAND.nextInt(25)); vector.setSafe(i, values[i].getBytes(Charsets.UTF_8)); } } vector.setValueCount(values.length); return values; }
static String[] populateVarCharValuesWithoutNull(VarCharVector vector, int size){ vector.allocateNew(); String values[] = new String[size]; for(int i =0; i < values.length; i++){ values[i] = RandomStringUtils.randomAlphanumeric(RAND.nextInt(25)); vector.setSafe(i, values[i].getBytes(Charsets.UTF_8)); } vector.setValueCount(values.length); return values; }
static void populate(VarCharVector vector, byte[][] values){ vector.allocateNew(); Random r = new Random(); for(int i =0; i < values.length; i++){ byte[] val = values[i]; if(val != null){ vector.setSafe(i, val, 0, val.length); } else { // add noise. this confirms that after pivot, noise is gone. byte[] bytes = new byte[r.nextInt(15)]; r.nextBytes(bytes); vector.setSafe(i, bytes, 0, bytes.length); vector.setNull(i); } } vector.setValueCount(values.length); }
private static Pair<VarCharVector, ResultVerifier> testVarCharVector(final int startIndexInCurrentOutput, final int startIndexInJob) { VarCharVector colVarCharV = new VarCharVector("colVarChar", allocator); colVarCharV.allocateNew(500, 5); colVarCharV.set(0, "value1".getBytes()); colVarCharV.set(1, "long long long long long long long long long long long long long long long long value".getBytes() ); colVarCharV.set(2, "long long long long value".getBytes()); colVarCharV.setNull(3); colVarCharV.set(4, "l".getBytes()); ResultVerifier verifier = new ResultVerifier() { @Override public void verify(DataPOJO output) { int index = startIndexInCurrentOutput; int uIndex = startIndexInJob; assertEquals("value1", output.extractValue("colVarChar", index)); assertNull(output.extractUrl("colVarChar", index++)); uIndex++; assertEquals("long long long long long long ", output.extractValue("colVarChar", index)); assertEquals(cellUrl(uIndex++, "colVarChar"), output.extractUrl("colVarChar", index++)); assertEquals("long long long long value", output.extractValue("colVarChar", index)); assertNull(output.extractUrl("colVarChar", index++)); assertNull(output.extractValue("colVarChar", index)); assertNull(output.extractUrl("colVarChar", index++)); assertEquals("l", output.extractValue("colVarChar", index)); assertNull(output.extractUrl("colVarChar", index++)); } }; return Pair.of(colVarCharV, verifier); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(valuesLength, metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner offset and value buffers */ vector.offsetBuffer.close(); vector.valueBuffer.close(); loadOffsetAndDataBuffer(valuesField, buffer.slice(0, valuesLength)); vector.setLastSet(metadata.getValueCount() - 1); }
){ in.allocateNew(count * 8, count);
@Test public void test() throws Exception { final VarCharVector varCharVector = new VarCharVector("field", allocator); varCharVector.allocateNew(10000, 1000);
@Test public void varcharRoundtrip() throws Exception { final int count = 1024; try ( VarCharVector in = new VarCharVector("in", allocator); VarCharVector out = new VarCharVector("out", allocator); ) { in.allocateNew(count * 8, count); for (int i = 0; i < count; i++) { if (i % 5 == 0) { byte[] data = ("hello-" + i).getBytes(Charsets.UTF_8); in.setSafe(i, data, 0, data.length); } } in.setValueCount(count); final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out)); try ( final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth()); final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount()); ) { fbv.ensureAvailableBlocks(count); Pivots.pivot(pivot, count, fbv, vbv); ValueVector[] ins = new ValueVector[]{in}; ValueVector[] outs = new ValueVector[]{out}; unpivotHelper(pivot, fbv, vbv, ins, outs, 0, count); unpivotHelper(pivot, fbv, vbv, ins, outs, 0, 100); unpivotHelper(pivot, fbv, vbv, ins, outs, 100, 924); } } }