/** * Initialize buffer to default size. */ public void initBuffer() { initBuffer(0); }
public void initBuffer(VectorizedRowBatch batch) { BytesColumnVector cv = (BytesColumnVector) batch.cols[outputColumnNum]; cv.isRepeating = false; cv.initBuffer(); }
@Override public void init() { initBuffer(0); }
protected void commonSetup() throws HiveException { /* * Make sure big table BytesColumnVectors have room for string values in the overflow batch... */ for (int column: bigTableByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } for (int column : nonOuterSmallTableKeyByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } for (int column : outerSmallTableKeyByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } for (int column: smallTableByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } batchCounter = 0; rowCounter = 0; }
public void firstBatchSetup(VectorizedRowBatch batch) throws HiveException { // Make sure small table BytesColumnVectors have room for string values in the big table and // overflow batchs... for (int column: smallTableByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) batch.cols[column]; bytesColumnVector.initBuffer(); } // Setup a scratch batch that will be used to play back big table rows that were spilled // to disk for the Hybrid Grace hash partitioning. spillReplayBatch = VectorizedBatchUtil.makeLike(batch); }
private void evaluateBytes(VectorizedRowBatch vrg) { BytesColumnVector cv = (BytesColumnVector) vrg.cols[outputColumnNum]; cv.isRepeating = true; cv.initBuffer(); if (!isNullValue) { cv.isNull[0] = false; cv.setVal(0, bytesValue, 0, bytesValueLength); } else { cv.isNull[0] = true; cv.noNulls = false; } }
private void evaluateBytes(ColumnVector colVector) { BytesColumnVector cv = (BytesColumnVector) colVector; cv.isRepeating = true; cv.initBuffer(); if (!isNullValue) { cv.isNull[0] = false; cv.setVal(0, bytesValue, 0, bytesValueLength); } else { cv.isNull[0] = true; cv.noNulls = false; } }
/** * Additional reset work for BytesColumnVector (releasing scratch bytes for by value strings). */ @Override public void reset() { super.reset(); initBuffer(0); }
private void evaluateBytes(VectorizedRowBatch vrg) { BytesColumnVector cv = (BytesColumnVector) vrg.cols[outputColumn]; cv.isRepeating = true; cv.noNulls = !isNullValue; cv.initBuffer(); if (!isNullValue) { cv.setVal(0, bytesValue, 0, bytesValueLength); } else { cv.isNull[0] = true; } }
private void evaluateBytes(VectorizedRowBatch vrg) { BytesColumnVector cv = (BytesColumnVector) vrg.cols[outputColumn]; cv.isRepeating = true; cv.noNulls = !isNullValue; cv.initBuffer(); if (!isNullValue) { cv.setVal(0, bytesValue, 0, bytesValueLength); cv.isNull[0] = false; } else { cv.isNull[0] = true; } }
protected void commonSetup(VectorizedRowBatch batch) throws HiveException { if (isLogDebugEnabled) { LOG.debug("VectorMapJoinInnerCommonOperator commonSetup begin..."); displayBatchColumns(batch, "batch"); displayBatchColumns(overflowBatch, "overflowBatch"); } // Make sure big table BytesColumnVectors have room for string values in the overflow batch... for (int column: bigTableByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } // Make sure small table BytesColumnVectors have room for string values in the big table and // overflow batchs... for (int column: smallTableByteColumnVectorColumns) { BytesColumnVector bytesColumnVector = (BytesColumnVector) batch.cols[column]; bytesColumnVector.initBuffer(); bytesColumnVector = (BytesColumnVector) overflowBatch.cols[column]; bytesColumnVector.initBuffer(); } // Setup a scratch batch that will be used to play back big table rows that were spilled // to disk for the Hybrid Grace hash partitioning. spillReplayBatch = VectorizedBatchUtil.makeLike(batch); }
public static VectorizedRowBatch getVectorizedRowBatchStringInLongOut() { VectorizedRowBatch batch = new VectorizedRowBatch(2); BytesColumnVector inV; LongColumnVector outV; inV = new BytesColumnVector(); outV = new LongColumnVector(); inV.initBuffer(); inV.setVal(0, StandardCharsets.UTF_8.encode("true").array()); inV.setVal(1, StandardCharsets.UTF_8.encode("TRUE").array()); inV.setVal(2, StandardCharsets.UTF_8.encode("TrUe").array()); inV.setVal(3, StandardCharsets.UTF_8.encode("false").array()); inV.setVal(4, StandardCharsets.UTF_8.encode("FALSE").array()); inV.setVal(5, StandardCharsets.UTF_8.encode("FaLsE").array()); inV.setVal(6, StandardCharsets.UTF_8.encode("").array()); inV.setVal(7, StandardCharsets.UTF_8.encode("Other").array()); batch.cols[0] = inV; batch.cols[1] = outV; batch.size = 8; return batch; }
private void testWriterText(TypeInfo type) throws HiveException { Text t1 = new Text("alpha"); Text t2 = new Text("beta"); BytesColumnVector bcv = new BytesColumnVector(vectorSize); bcv.noNulls = false; bcv.initBuffer(); bcv.setVal(0, t1.getBytes(), 0, t1.getLength()); bcv.isNull[1] = true; bcv.setVal(2, t2.getBytes(), 0, t2.getLength()); bcv.isNull[3] = true; bcv.setVal(4, t1.getBytes(), 0, t1.getLength()); VectorExpressionWriter vew = getWriter(type); for (int i = 0; i < vectorSize; i++) { Writable w = (Writable) vew.writeValue(bcv, i); if (w != null) { byte [] val = new byte[bcv.length[i]]; System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]); Writable expected = getWritableValue(type, val); Assert.assertEquals(expected, w); } else { Assert.assertTrue(bcv.isNull[i]); } } }
private void testSetterText(TypeInfo type) throws HiveException { Text t1 = new Text("alpha"); Text t2 = new Text("beta"); BytesColumnVector bcv = new BytesColumnVector(vectorSize); bcv.noNulls = false; bcv.initBuffer(); bcv.setVal(0, t1.getBytes(), 0, t1.getLength()); bcv.isNull[1] = true; bcv.setVal(2, t2.getBytes(), 0, t2.getLength()); bcv.isNull[3] = true; bcv.setVal(4, t1.getBytes(), 0, t1.getLength()); Object[] values = new Object[this.vectorSize]; VectorExpressionWriter vew = getWriter(type); for (int i = 0; i < vectorSize; i++) { values[i] = null; // setValue() should be able to handle null input Writable w = (Writable) vew.setValue(values[i], bcv, i); if (w != null) { byte [] val = new byte[bcv.length[i]]; System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]); Writable expected = getWritableValue(type, val); Assert.assertEquals(expected, w); } else { Assert.assertTrue(bcv.isNull[i]); } } }
@Test // Load a BytesColumnVector by copying in large data, enough to force // the buffer to expand. public void testLoadBytesColumnVectorByValueLargeData() { BytesColumnVector bcv = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE); bcv.initBuffer(10); // initialize with estimated element size 10 // Record initial buffer size int initialBufferSize = bcv.bufferSize(); String s = "0123456789"; while (s.length() < 500) { s += s; } byte[] b = s.getBytes(StandardCharsets.UTF_8); for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) { bcv.setVal(i, b, 0, b.length); } // Current buffer size should be larger than initial size Assert.assertTrue(bcv.bufferSize() > initialBufferSize); }
public static VectorizedRowBatch getBatchForStringMath() { VectorizedRowBatch batch = new VectorizedRowBatch(3); LongColumnVector inL; BytesColumnVector inS, outS; inL = new LongColumnVector(); inS = new BytesColumnVector(); outS = new BytesColumnVector(); inL.vector[0] = 0; inL.vector[1] = 255; inL.vector[2] = 0; inS.initBuffer(); inS.setVal(0, "00".getBytes(StandardCharsets.UTF_8), 0, 2); inS.setVal(1, "3232".getBytes(StandardCharsets.UTF_8), 0, 4); byte[] bad = "bad data".getBytes(StandardCharsets.UTF_8); inS.setVal(2, bad, 0, bad.length); batch.cols[0] = inS; batch.cols[1] = inL; batch.cols[2] = outS; batch.size = 3; return batch; }
private VectorizedRowBatch getBatchStrDblLongWithStrOut() { VectorizedRowBatch b = new VectorizedRowBatch(4); BytesColumnVector strCol = new BytesColumnVector(); LongColumnVector longCol = new LongColumnVector(); DoubleColumnVector dblCol = new DoubleColumnVector(); BytesColumnVector outCol = new BytesColumnVector(); b.cols[0] = strCol; b.cols[1] = longCol; b.cols[2] = dblCol; b.cols[3] = outCol; strCol.initBuffer(); strCol.setVal(0, blue, 0, blue.length); strCol.setVal(1, red, 0, red.length); longCol.vector[0] = 0; longCol.vector[1] = 1; dblCol.vector[0] = 0.0; dblCol.vector[1] = 1.0; // set one null value for possible later use longCol.isNull[1] = true; // but have no nulls initially longCol.noNulls = true; strCol.noNulls = true; dblCol.noNulls = true; outCol.initBuffer(); b.size = 2; return b; }
VectorizedRowBatch makeStringBatchMixedCase() { // create a batch with two string ("Bytes") columns VectorizedRowBatch batch = new VectorizedRowBatch(2, VectorizedRowBatch.DEFAULT_SIZE); BytesColumnVector v = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE); batch.cols[0] = v; BytesColumnVector outV = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE); outV.initBuffer(); batch.cols[1] = outV; /* * Add these 3 values: * * mixedUp * green * NULL */ v.setRef(0, mixedUp, 0, mixedUp.length); v.isNull[0] = false; v.setRef(1, green, 0, green.length); v.isNull[1] = false; v.setRef(2, emptyString, 0, emptyString.length); v.isNull[2] = true; v.noNulls = false; batch.size = 3; return batch; }
private VectorizedRowBatch getBatchStringDecimal() { VectorizedRowBatch b = new VectorizedRowBatch(2); BytesColumnVector bv; b.cols[0] = bv = new BytesColumnVector(); b.cols[1] = new DecimalColumnVector(18, 2); bv.initBuffer(); byte[] x0 = toBytes("1.10"); byte[] x1 = toBytes("-2.20"); byte[] x2 = toBytes("99999999999999.0"); bv.setVal(0, x0, 0, x0.length); bv.setVal(1, x1, 0, x1.length); bv.setVal(2, x2, 0, x2.length); return b; }
@Test public void testFilterStringNotBetween() throws HiveException { // Spot check only. Non-standard cases are checked for the same template in another test. int seed = 17; VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( 3, 2, seed); vrb.cols[0] = new BytesColumnVector(); BytesColumnVector bcv = (BytesColumnVector) vrb.cols[0]; bcv.initBuffer(); bcv.setVal(0, a, 0, 1); bcv.setVal(1, b, 0, 1); bcv.setVal(2, c, 0, 1); VectorExpression expr = new FilterStringColumnNotBetween(0, b, c); expr.evaluate(vrb); assertEquals(1, vrb.size); assertTrue(vrb.selectedInUse); assertEquals(0, vrb.selected[0]); }