RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, RCFile.createMetadata(new Text("apple"), new Text("block"), bytes.set(i, cu); writer.append(bytes); bytes.clear(); for (int i = 0; i < record_2.length; i++) { bytes.set(i, cu); writer.append(bytes); writer.close();
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
CompressionOutputStream deflateFilter = null; DataOutputStream deflateOut = null; boolean isCompressed = isCompressed(); int valueLength = 0; if (isCompressed) { writeKey(key, keyLength + valueLength, keyLength); clearColumnBuffers();
CompressionOutputStream deflateFilter = null; DataOutputStream deflateOut = null; boolean isCompressed = isCompressed(); int valueLength = 0; if (isCompressed) { writeKey(key, keyLength + valueLength, keyLength); clearColumnBuffers();
if (isCompressed()) { compressionBuffer[columnIndex].reset(); deflateFilter[columnIndex].resetState(); checkAndWriteSync(); // sync if (!isCompressed()) { out.writeInt(keyLength); clearColumnBuffers();
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec); outWriter.flushBlock(key.getKeyBuffer(), value.getValueBuffer(), recordLength, keyLength, key.getCompressedKeyLength()); } catch (Throwable e) {
init(conf, fs.create(name, true, bufferSize, replication, blockSize, progress), codec, metadata); initializeFileHeader(); writeFileHeader(); finalizeFileHeader(); key = new KeyBuffer(columnNumber);
RCFile.Writer writer = new RCFile.Writer(target.getFileSystem(conf), conf, target, () -> {}, codec); return new ExtendedRecordWriter()
final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);
init(name, conf, fs.create(name, true, bufferSize, replication, blockSize, progress), codec, metadata); initializeFileHeader(); writeFileHeader(); finalizeFileHeader(); key = new KeyBuffer(columnNumber); value = new ValueBuffer(key);
final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);
final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec); outWriter.flushBlock(key.getKeyBuffer(), value.getValueBuffer(), key.getRecordLength(), key.getKeyLength(), key.getCompressedKeyLength());
final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);
if ((columnBufferSize > columnsBufferSize) || (bufferedRecords >= RECORD_INTERVAL)) { flushRecords();
if ((columnBufferSize > columnsBufferSize) || (bufferedRecords >= RECORD_INTERVAL)) { flushRecords();
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
/** {@inheritDoc} */ @Override public RecordWriter<WritableComparable, BytesRefArrayWritable> getRecordWriter( FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { Path outputPath = getWorkOutputPath(job); FileSystem fs = outputPath.getFileSystem(job); if (!fs.exists(outputPath)) { fs.mkdirs(outputPath); } Path file = new Path(outputPath, name); CompressionCodec codec = null; if (getCompressOutput(job)) { Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); } final RCFile.Writer out = new RCFile.Writer(fs, job, file, progress, codec); return new RecordWriter<WritableComparable, BytesRefArrayWritable>() { @Override public void close(Reporter reporter) throws IOException { out.close(); } @Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } }; }
/** {@inheritDoc} */ @Override public RecordWriter<WritableComparable, BytesRefArrayWritable> getRecordWriter( FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { Path outputPath = getWorkOutputPath(job); FileSystem fs = outputPath.getFileSystem(job); Path file = new Path(outputPath, name); CompressionCodec codec = null; if (getCompressOutput(job)) { Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); } final RCFile.Writer out = new RCFile.Writer(fs, job, file, progress, codec); return new RecordWriter<WritableComparable, BytesRefArrayWritable>() { @Override public void close(Reporter reporter) throws IOException { out.close(); } @Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } }; }
public synchronized void close() throws IOException { if (bufferedRecords > 0) { flushRecords(); } clearColumnBuffers(); if (isCompressed()) { for (int i = 0; i < columnNumber; i++) { deflateFilter[i].close(); IOUtils.closeStream(deflateOut[i]); } keyDeflateFilter.close(); IOUtils.closeStream(keyDeflateOut); CodecPool.returnCompressor(keyCompressor); keyCompressor = null; CodecPool.returnCompressor(compressor); compressor = null; } if (out != null) { // Close the underlying stream if we own it... out.flush(); out.close(); out = null; } } }