Refine search
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
"hive and hadoop".getBytes(StandardCharsets.UTF_8), new byte[0], "NULL".getBytes(StandardCharsets.UTF_8)}; s = new BytesRefArrayWritable(bytesArray.length); s.set(0, new BytesRefWritable("123".getBytes(StandardCharsets.UTF_8))); s.set(1, new BytesRefWritable("456".getBytes(StandardCharsets.UTF_8))); s.set(2, new BytesRefWritable("789".getBytes(StandardCharsets.UTF_8))); s.set(3, new BytesRefWritable("1000".getBytes(StandardCharsets.UTF_8))); s.set(4, new BytesRefWritable("5.3".getBytes(StandardCharsets.UTF_8))); s.set(5, new BytesRefWritable("hive and hadoop".getBytes(StandardCharsets.UTF_8))); s.set(6, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); s.set(7, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); patialS.set(0, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); patialS.set(1, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); patialS.set(2, new BytesRefWritable("789".getBytes(StandardCharsets.UTF_8))); patialS.set(3, new BytesRefWritable("1000".getBytes(StandardCharsets.UTF_8))); patialS.set(4, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); patialS.set(5, new BytesRefWritable("".getBytes(StandardCharsets.UTF_8))); patialS.set(6, new BytesRefWritable("NULL".getBytes(StandardCharsets.UTF_8))); patialS.set(7, new BytesRefWritable("".getBytes(StandardCharsets.UTF_8)));
rest = new BytesRefArrayWritable(); rest.resetValid(recordsNumInValBuffer); int length = selCol.prvLength; BytesRefWritable currentCell = rest.get(i); currentCell.set(uncompData, columnNextRowStart, length); } else { currentCell.set(decompCallBack, columnNextRowStart, length);
/** {@inheritDoc} */ @Override public int compareTo(BytesRefWritable other) { if (other == null) { throw new IllegalArgumentException("Argument can not be null."); } if (this == other) { return 0; } try { return WritableComparator.compareBytes(getData(), start, getLength(), other.getData(), other.start, other.getLength()); } catch (IOException e) { throw new RuntimeException(e); } }
public void append(BytesRefWritable data) throws IOException { data.writeDataTo(columnValBuffer); int currentLen = data.getLength(); if (prevValueLength < 0) { startNewGroup(currentLen); return; } if (currentLen != prevValueLength) { flushGroup(); startNewGroup(currentLen); } else { runLength++; } }
"NULL".getBytes(StandardCharsets.UTF_8)}; BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length); for (int i = 0; i < record_1.length; i++) { BytesRefWritable cu = new BytesRefWritable(record_1[i], 0, record_1[i].length); bytes.set(i, cu); bytes.clear(); for (int i = 0; i < record_2.length; i++) { BytesRefWritable cu = new BytesRefWritable(record_2[i], 0, record_2[i].length); bytes.set(i, cu); for (int row=0; row<result.size(); row++) { brw = result.get(row); int start = brw.getStart(); int len = brw.getLength(); byte[] actualData = Arrays.copyOfRange(brw.getData(), start, start + len); byte[] expectedData = (row == 0) ? record_1[col] : record_2[col]; assertArrayEquals("col="+col+" : row="+row, expectedData, actualData);
cachedByteArrayRef.setData(rawBytesField.getData()); } catch (IOException e) { throw new RuntimeException(e); int byteLength = getLength(objectInspector, cachedByteArrayRef, rawBytesField.getStart(), rawBytesField.getLength()); if (byteLength == -1) { return null; field.init(cachedByteArrayRef, rawBytesField.getStart(), byteLength); return field.getObject(); } else { if (getLength(objectInspector, cachedByteArrayRef, rawBytesField.getStart(), rawBytesField.getLength()) == -1) { return null;
currentValueBuffer(); ret.resetValid(columnNumber); } else { if (!currentValue.inited) { ret.resetValid(columnNumber); ret.set(i, null); } else { BytesRefWritable ref = ret.unCheckedGet(i); ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), col.rowReadIndex, col.prvLength); } else { ref.set(currentValue.lazyDecompressCallbackObjs[j], col.rowReadIndex, col.prvLength); ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), col.rowReadIndex, col.prvLength); col.rowReadIndex += col.prvLength;
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length); final int recCount = 100; Random rand = new Random(); BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length); bytes.set(i, cu); bytes.clear();
int size = columns.size(); for (int i = 0; i < size; i++) { BytesRefWritable cu = columns.get(i); int plainLen = cu.getLength(); columnBufferSize += plainLen; columnValuePlainLength[i] += plainLen; for (int i = columns.size(); i < columnNumber; i++) { columnBuffers[i].append(BytesRefWritable.ZeroBytesRefWritable);
/** * Return the uncompressed size of this field */ public long getSerializedSize() { if (rawBytesField == null) { return 0; } return rawBytesField.getLength(); }
protected void initialize(int size) throws SerDeException { field = new BytesRefWritable[size]; for (int i = 0; i < size; i++) { field[i] = new BytesRefWritable(); serializeCache.set(i, field[i]); } serializedSize = 0; stats = new SerDeStats(); lastOperationSerialize = false; lastOperationDeserialize = false; }
private void nextRandomRow(byte[][] row, BytesRefArrayWritable bytes) { bytes.resetValid(row.length); for (int i = 0; i < row.length; i++) { int len = Math.abs(randColLenGenerator.nextInt(columnMaxSize)); row[i] = new byte[len]; for (int j = 0; j < len; j++) { row[i][j] = getRandomChar(randomCharGenerator); } bytes.get(i).set(row[i], 0, len); } }
ret.resetValid(columnNumber); int i = prjColIDs[j]; BytesRefWritable ref = ret.unCheckedGet(i); ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), columnCurrentRowStart, length); } else { ref.set(currentValue.lazyDecompressCallbackObjs[j], columnCurrentRowStart, length);
LazyBinarySerDe.serialize(serializeStream, f, foi, true, warnedOnceNullMapKey); field[i].set(serializeStream.getData(), streamOffset, serializeStream .getLength() - streamOffset);
/** * Returns the underlying bytes. * * @throws IOException */ public byte[] getData() throws IOException { lazyDecompress(); return bytes; }
@Override public Writable newInstance() { return new BytesRefWritable(); }
/** {@inheritDoc} */ @Override public boolean equals(Object right_obj) { if (right_obj == null || !(right_obj instanceof BytesRefWritable)) { return false; } return compareTo((BytesRefWritable) right_obj) == 0; }
private Writable updateColumnValue(RCFileColumn column, BytesRefWritable bytesRef) throws IOException { if(bytesRef.getLength() == 0) { // This is a null field. return NullWritable.get(); } Writable newColumnValue = column.newWritable(); // Small optimization to bypass DataInput read if the column writable is // BytesRefWritable if (newColumnValue.getClass() == BytesRefWritable.class) { newColumnValue = bytesRef; } else { byte[] currentRowBytes = Arrays.copyOfRange(bytesRef.getData(), bytesRef.getStart(), bytesRef.getStart() + bytesRef.getLength()); DataInput dataInput = ByteStreams.newDataInput(currentRowBytes); newColumnValue.readFields(dataInput); } return newColumnValue; }