writer.append(bytes);
bytes.set(i, cu); writer.append(bytes); bytes.clear();
writer.append(bytes);
@Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } };
@Override public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException { out.append(value); }
@Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } };
@Override public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException { out.append(value); }
@Override public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException { out.append(value); }
@Override public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException { out.append(value); }
System.out.println("write position:" + writer.getLength()); writer.append(bytes);
private void writeRCFileTest(FileSystem fs, int rowCount, Path file, int columnNum, CompressionCodec codec) throws IOException { fs.delete(file, true); resetRandomGenerators(); RCFileOutputFormat.setColumnNumber(conf, columnNum); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, codec); byte[][] columnRandom; BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnNum); columnRandom = new byte[columnNum][]; for (int i = 0; i < columnNum; i++) { BytesRefWritable cu = new BytesRefWritable(); bytes.set(i, cu); } for (int i = 0; i < rowCount; i++) { nextRandomRow(columnRandom, bytes); writer.append(bytes); } writer.close(); }
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
@Override public void write(Writable r) throws IOException { outWriter.append(r); }
@Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } };
@Override public void write(Writable r) throws IOException { outWriter.append(r); }
@Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } };
@Override public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException { out.append(value); }
@Override public void write(Writable value) throws IOException { writer.append(value); length = writer.getLength(); }