Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(1000) .compress(CompressionKind.NONE) .batchSize(1000) .bufferSize(100) .blockPadding(false)); OrcStruct row = new OrcStruct(3); OrcUnion union = new OrcUnion();
Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .compress(CompressionKind.NONE) .stripeSize(50000) .bufferSize(100) .rowIndexStride(0) .memory(memory) .batchSize(100) .version(OrcFile.Version.V_0_12)); assertEquals(testFilePath, memory.path); for(int i=0; i < 2500; ++i) {
.fileSystem(fs) .setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE) .callback(indexBuilder) .stripeSize(128);
.compress(compression) .version(version) .rowIndexStride(rowIndexStride) .inspector(reader.getObjectInspector())); if (isLogDebugEnabled) { LOG.info("ORC merge file output path: " + outPath);
writerOptions = OrcFile.writerOptions(options.getConfiguration()); writerOptions.fileSystem(fs).callback(indexBuilder); if (!options.isWritingBase()) { writerOptions.blockPadding(false); writerOptions.bufferSize(DELTA_BUFFER_SIZE); writerOptions.stripeSize(DELTA_STRIPE_SIZE); writerOptions.inspector(createEventSchema(findRecId(options.getInspector(), options.getRecordIdColumn()))); this.writer = OrcFile.createWriter(this.path, writerOptions);
if ((propVal = getSettingFromPropsFallingBackToConf( OrcFile.OrcTableProperties.STRIPE_SIZE.getPropName(),props,conf)) != null){ options.stripeSize(Long.parseLong(propVal)); options.compress(CompressionKind.valueOf(propVal)); options.bufferSize(Integer.parseInt(propVal)); options.rowIndexStride(Integer.parseInt(propVal)); OrcFile.OrcTableProperties.ENABLE_INDEXES.getPropName(),props,conf)) != null){ if ("false".equalsIgnoreCase(propVal)) { options.rowIndexStride(0); options.blockPadding(Boolean.parseBoolean(propVal)); options.encodingStrategy(EncodingStrategy.valueOf(propVal)); options.bloomFilterColumns(propVal); options.bloomFilterFpp(Double.parseDouble(propVal));
.memory(new NullMemoryManager()) .fileSystem(fileSystem) .compress(reader.getCompression()) .inspector(reader.getObjectInspector());
OrcFile.writerOptions(options.getConfiguration()); if (!options.isWritingBase()) { opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE) .stripeSize(OrcRecordUpdater.DELTA_STRIPE_SIZE) .blockPadding(false) .compress(CompressionKind.NONE) .rowIndexStride(0); opts.inspector(options.getInspector()) .callback(watcher); final Writer writer = OrcFile.createWriter(filename, opts); return new org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter() {
Counter counter) throws IOException, InterruptedException { WriterOptions options = OrcFile.writerOptions(getConf()); options.fileSystem(fileSystem); options.inspector(new DataModelInspector(getDataModelDescriptor())); options.version(formatVersion); options.compress(compressionKind); options.stripeSize(stripeSize);
return createWriter(path, writerOptions(conf) .fileSystem(fs) .inspector(inspector) .stripeSize(stripeSize) .compress(compress) .bufferSize(bufferSize) .rowIndexStride(rowIndexStride));
private static RecordWriter createRecordWriter(Path target, List<Long> columnIds, List<Type> columnTypes, boolean writeMetadata) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { OrcFile.WriterOptions options = new OrcWriterOptions(CONFIGURATION) .memory(new NullMemoryManager(CONFIGURATION)) .fileSystem(fileSystem) .compress(SNAPPY); if (writeMetadata) { options.callback(createFileMetadataCallback(columnIds, columnTypes)); } return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException | IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); } }
private static RecordWriter createRecordWriter(Path target, List<Long> columnIds, List<Type> columnTypes, boolean writeMetadata) { try (FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { OrcFile.WriterOptions options = OrcFile.writerOptions(CONFIGURATION) .memory(new NullMemoryManager()) .fileSystem(fileSystem) .compress(SNAPPY); if (writeMetadata) { options.callback(createFileMetadataCallback(columnIds, columnTypes)); } return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException | IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); } }
@Override public void close(boolean b) throws IOException { // if we haven't written any rows, we need to create a file with a // generic schema. if (writer == null) { // a row with no columns ObjectInspector inspector = ObjectInspectorFactory. getStandardStructObjectInspector(new ArrayList<String>(), new ArrayList<ObjectInspector>()); options.inspector(inspector); writer = OrcFile.createWriter(path, options); } writer.close(); }
private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = new OrcWriterOptions(conf) .memory(new NullMemoryManager(conf)) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }
private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }
private static RecordWriter createRecordWriter(Path target, Configuration conf) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { WriterOptions options = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .compress(ZLIB); try { return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } }
private static RecordWriter createRecordWriter(Path target, Configuration conf) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { WriterOptions options = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .compress(ZLIB); try { return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } }
@Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { if (writer == null) { // a row with no columns ObjectInspector inspector = ObjectInspectorFactory. getStandardStructObjectInspector(new ArrayList<String>(), new ArrayList<ObjectInspector>()); options.inspector(inspector); writer = OrcFile.createWriter(path, options); } writer.close(); } }
private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }