public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
@Test public void testReadOldFileHeader() throws IOException { String[] row = new String[]{"Tester", "Bart", "333 X St.", "Reno", "NV", "USA"}; RCFile.Reader reader = new RCFile.Reader(fs, new Path(HiveTestUtils.getFileFromClasspath("rc-file-v0.rc")), conf); LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); assertTrue("old file reader first row", reader.next(rowID)); reader.getCurrentRow(cols); assertEquals(row.length, cols.size()); for (int i=0; i < cols.size(); ++i) { assertEquals(row[i], new String(cols.get(i).getBytesCopy())); } assertFalse("old file reader end", reader.next(rowID)); reader.close(); }
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
public void init(BytesRefArrayWritable cols) { for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; if (fieldIndex < cols.size()) { fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex)); } else { // select columns that actually do not exist in the file. fieldInfoList[fieldIndex].init(null); } } }
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
/** * Print record to string builder * @param value * @param buf * @throws IOException */ private void printRecord(BytesRefArrayWritable value, StringBuilder buf) throws IOException { int n = value.size(); if (n > 0) { BytesRefWritable v = value.unCheckedGet(0); ByteBuffer bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); for (int i = 1; i < n; i++) { // do not put the TAB for the last column buf.append(RCFileCat.TAB); v = value.unCheckedGet(i); bb = ByteBuffer.wrap(v.getData(), v.getStart(), v.getLength()); buf.append(decoder.decode(bb)); } buf.append(RCFileCat.NEWLINE); } }
private void parseStringColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true; if (hiveColumnIndexes[column] >= value.size()) { // this partition may contain fewer fields than what's declared in the schema // this happens when additional columns are added to the hive table after a partition has been created nulls[column] = true; } else { BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]); byte[] bytes; try { bytes = fieldData.getData(); } catch (IOException e) { throw Throwables.propagate(e); } int start = fieldData.getStart(); int length = fieldData.getLength(); parseStringColumn(column, bytes, start, length); } }
private void parseBooleanColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true; if (hiveColumnIndexes[column] >= value.size()) { // this partition may contain fewer fields than what's declared in the schema // this happens when additional columns are added to the hive table after a partition has been created nulls[column] = true; } else { BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]); byte[] bytes; try { bytes = fieldData.getData(); } catch (IOException e) { throw Throwables.propagate(e); } int start = fieldData.getStart(); int length = fieldData.getLength(); parseBooleanColumn(column, bytes, start, length); } }
private void parseLongColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true; if (hiveColumnIndexes[column] >= value.size()) { // this partition may contain fewer fields than what's declared in the schema // this happens when additional columns are added to the hive table after a partition has been created nulls[column] = true; } else { BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]); byte[] bytes; try { bytes = fieldData.getData(); } catch (IOException e) { throw Throwables.propagate(e); } int start = fieldData.getStart(); int length = fieldData.getLength(); parseLongColumn(column, bytes, start, length); } }
private void parseDoubleColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true; if (hiveColumnIndexes[column] >= value.size()) { // this partition may contain fewer fields than what's declared in the schema // this happens when additional columns are added to the hive table after a partition has been created nulls[column] = true; } else { BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]); byte[] bytes; try { bytes = fieldData.getData(); } catch (IOException e) { throw Throwables.propagate(e); } int start = fieldData.getStart(); int length = fieldData.getLength(); parseDoubleColumn(column, bytes, start, length); } }