@Override public void write(NullWritable key, OrcSerdeRow row) throws IOException, InterruptedException { if (writer == null) { options.inspector(row.getInspector()); writer = OrcFile.createWriter(path, options); } writer.addRow(row.getRow()); }
public ByteBuffer getMetadataToCache( FileSystem fs, Path path, ByteBuffer[] addedVals) throws IOException { // For now, there's nothing special to return in addedVals. Just return the footer. return OrcFile.createReader(fs, path).getSerializedFileFooter(); } }
@Override public StatsProvidingRecordWriter getHiveRecordWriter(JobConf conf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable reporter) throws IOException { return new OrcRecordWriter(path, getOptions(conf, tableProperties)); }
static WriterOptions createOrcWriterOptions(ObjectInspector sourceOi, Configuration conf, CacheWriter cacheWriter, int allocSize) throws IOException { return OrcFile.writerOptions(conf).stripeSize(Long.MAX_VALUE).blockSize(Long.MAX_VALUE) .rowIndexStride(Integer.MAX_VALUE) // For now, do not limit this - one RG per split .blockPadding(false).compress(CompressionKind.NONE).version(Version.CURRENT) .encodingStrategy(EncodingStrategy.SPEED).bloomFilterColumns(null).inspector(sourceOi) .physicalWriter(cacheWriter).memory(MEMORY_MANAGER).bufferSize(allocSize); }
private void writeFile(ObjectInspector inspector, Path filePath) throws IOException { Writer writer = OrcFile.createWriter( fs, filePath, conf, inspector, 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); writer.close(); }
@Override @SuppressWarnings("deprecation") public Serializer createSerializer() { return new OrcSerde(); } },
@Override public void next(OrcStruct next) throws IOException { if(!nextFromCurrentFile(next)) { //only have 1 file so done nextRecord = null; getRecordReader().close(); } } }
@Override public Void run() throws Exception { OrcInputFormat.generateSplitsInfo(conf, new Context(conf, -1, null)); return null; } });
@Override void writeStripe(OrcProto.StripeFooter.Builder builder, int requiredIndexEntries) throws IOException { super.writeStripe(builder, requiredIndexEntries); stream.flush(); length.flush(); recordPosition(rowIndexPosition); }
private void writeFileStatistics(OrcProto.Footer.Builder builder, TreeWriter writer) throws IOException { builder.addStatistics(writer.fileStatistics.serialize()); for (TreeWriter child : writer.getChildrenWriters()) { writeFileStatistics(builder, child); } }
public OrcInputFormat.AcidDirInfo createAdi( OrcInputFormat.Context context, MockFileSystem fs, String path) throws IOException { return new OrcInputFormat.FileGenerator( context, fs, new MockPath(fs, path), false, null).call(); }
private BigRow createRandomRow(long[] intValues, double[] doubleValues, String[] stringValues, BytesWritable[] byteValues, String[] words, int i) { InnerStruct inner = new InnerStruct((int) intValues[i], stringValues[i]); InnerStruct inner2 = new InnerStruct((int) (intValues[i] >> 32), words[i % words.length] + "-x"); return new BigRow((intValues[i] & 1) == 0, (byte) intValues[i], (short) intValues[i], (int) intValues[i], intValues[i], (float) doubleValues[i], doubleValues[i], byteValues[i],stringValues[i], new MiddleStruct(inner, inner2), list(), map(inner,inner2)); }
private List<OrcInputFormat.SplitStrategy<?>> createSplitStrategies( OrcInputFormat.Context context, OrcInputFormat.FileGenerator gen) throws IOException { OrcInputFormat.AcidDirInfo adi = gen.call(); return OrcInputFormat.determineSplitStrategies( null, context, adi.fs, adi.splitPath, adi.baseFiles, adi.deleteEvents, null, null, true); }
public List<SplitStrategy<?>> createOrCombineStrategies(OrcInputFormat.Context context, MockFileSystem fs, String path, OrcInputFormat.CombinedCtx combineCtx) throws IOException { OrcInputFormat.AcidDirInfo adi = createAdi(context, fs, path); return OrcInputFormat.determineSplitStrategies(combineCtx, context, adi.fs, adi.splitPath, adi.baseFiles, adi.deleteEvents, null, null, true); }
@Override public void write(NullWritable nullWritable, OrcSerdeRow row) throws IOException { if (writer == null) { options.inspector(row.getInspector()); writer = OrcFile.createWriter(path, options); } writer.addRow(row.getRow()); }
@Override @SuppressWarnings("deprecation") public Serializer createSerializer() { return new OrcSerde(); } },
@Override public void write(Writable row) throws IOException { OrcSerdeRow serdeRow = (OrcSerdeRow) row; if (writer == null) { options.inspector(serdeRow.getInspector()); writer = OrcFile.createWriter(path, options); } writer.addRow(serdeRow.getRow()); }
@Override public void write(NullWritable nullWritable, OrcSerdeRow row) throws IOException { if (writer == null) { options.inspector(row.getInspector()); writer = OrcFile.createWriter(path, options); } writer.addRow(row.getRow()); }
@Override public void write(NullWritable key, OrcSerdeRow row) throws IOException, InterruptedException { if (writer == null) { options.inspector(row.getInspector()); writer = OrcFile.createWriter(path, options); } writer.addRow(row.getRow()); }