WriterOptions(Properties tableProperties, Configuration conf) { super(tableProperties, conf); useUTCTimestamp(true); if (conf.getBoolean(HiveConf.ConfVars.HIVE_ORC_WRITER_LLAP_MEMORY_MANAGER_ENABLED.varname, true) && LlapProxy.isDaemon()) { memory(getThreadLocalOrcLlapMemoryManager(conf)); } }
.bufferSize(100) .rowIndexStride(0) .memory(memory) .batchSize(100) .version(OrcFile.Version.V_0_11));
.bufferSize(100) .rowIndexStride(0) .memory(memory) .batchSize(100) .version(OrcFile.Version.V_0_12));
.memory(new NullMemoryManager()) .fileSystem(fileSystem) .compress(reader.getCompression())
private static RecordWriter createRecordWriter(Path target, List<Long> columnIds, List<Type> columnTypes, boolean writeMetadata) { try (FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { OrcFile.WriterOptions options = OrcFile.writerOptions(CONFIGURATION) .memory(new NullMemoryManager()) .fileSystem(fileSystem) .compress(SNAPPY); if (writeMetadata) { options.callback(createFileMetadataCallback(columnIds, columnTypes)); } return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException | IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); } }
private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }
private static RecordWriter createRecordWriter(Path target, Configuration conf) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { WriterOptions options = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .compress(ZLIB); try { return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } }
private static RecordWriter createRecordWriter(Path target, Configuration conf) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { WriterOptions options = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .compress(ZLIB); try { return WRITER_CONSTRUCTOR.newInstance(target, options); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } }
private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(conf) .memory(new NullMemoryManager()) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }
static WriterOptions createOrcWriterOptions(ObjectInspector sourceOi, Configuration conf, CacheWriter cacheWriter, int allocSize) throws IOException { return OrcFile.writerOptions(conf).stripeSize(Long.MAX_VALUE).blockSize(Long.MAX_VALUE) .rowIndexStride(Integer.MAX_VALUE) // For now, do not limit this - one RG per split .blockPadding(false).compress(CompressionKind.NONE).version(Version.CURRENT) .encodingStrategy(EncodingStrategy.SPEED).bloomFilterColumns(null).inspector(sourceOi) .physicalWriter(cacheWriter).memory(MEMORY_MANAGER).bufferSize(allocSize); }
@Override public OrcWriterOptions memory(MemoryManager value) { super.memory(value); return this; } }
OrcFile.writerOptions(conf).inspector(inspector).fileSystem(fs) .blockPadding(false).bufferSize(10000).compress(CompressionKind.NONE) .stripeSize(1).memory(mgr).batchSize(2).version(OrcFile.Version.V_0_11)); String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3", "2.0", "2.1", "3.0", "ignore.4", "ignore.5", "ignore.6"};
.memory(mgr).batchSize(2)); options.finalDestination(root); RecordUpdater ru = of.getRecordUpdater(root, options);
.blockSize(blockSizeValue) .bloomFilterFpp(bloomFilterFpp) .memory(getMemoryManager(conf)) .inspector(inspector) .stripeSize(stripeSize)
static WriterOptions createOrcWriterOptions(ObjectInspector sourceOi, Configuration conf, CacheWriter cacheWriter, int allocSize) throws IOException { return OrcFile.writerOptions(conf).stripeSize(Long.MAX_VALUE).blockSize(Long.MAX_VALUE) .rowIndexStride(Integer.MAX_VALUE) // For now, do not limit this - one RG per split .blockPadding(false).compress(CompressionKind.NONE).version(Version.CURRENT) .encodingStrategy(EncodingStrategy.SPEED).bloomFilterColumns(null).inspector(sourceOi) .physicalWriter(cacheWriter).memory(MEMORY_MANAGER).bufferSize(allocSize); }