private static void createFileWithOnlyUserMetadata(File file, Map<String, String> metadata) throws IOException { Configuration conf = new Configuration(); OrcFile.WriterOptions writerOptions = new OrcWriterOptions(conf) .memory(new NullMemoryManager(conf)) .inspector(createSettableStructObjectInspector("test", BIGINT)) .compress(SNAPPY); Writer writer = OrcFile.createWriter(new Path(file.toURI()), writerOptions); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(entry.getValue().getBytes(UTF_8))); } writer.close(); }
@Test public void emptyFile() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(1000) .compress(CompressionKind.NONE) .bufferSize(100)); writer.close(); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); assertEquals(false, reader.rows().hasNext()); assertEquals(CompressionKind.NONE, reader.getCompression()); assertEquals(0, reader.getNumberOfRows()); assertEquals(0, reader.getCompressionSize()); assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); assertEquals(3, reader.getContentLength()); assertEquals(false, reader.getStripes().iterator().hasNext()); }
.inspector(inspector) .stripeSize(1000) .compress(CompressionKind.NONE) .bufferSize(100) .rowIndexStride(1000));
.fileSystem(fs) .setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE)); VectorizedRowBatch batch = fileSchema.createRowBatch(1000); batch.size = 1000;
.fileSystem(fs) .setSchema(fileSchema) .compress(org.apache.orc.CompressionKind.NONE)); VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000); batch.size = 1000;
.inspector(inspector) .stripeSize(1000) .compress(CompressionKind.NONE) .bufferSize(100)); writer.addUserMetadata("my.meta", byteBuf(1, 2, 3, 4, 5, 6, 7, -1, -2, 127,
.fileSystem(fs) .inspector(OrcStruct.createObjectInspector(typeInfo)) .compress(CompressionKind.NONE) .callback(indexBuilder) .stripeSize(128));
.stripeSize(stripeSize) .bufferSize(bufferSize) .compress(compress);
.stripeSize(OrcRecordUpdater.DELTA_STRIPE_SIZE) .blockPadding(false) .compress(CompressionKind.NONE) .rowIndexStride(0);
.inspector(inspector) .stripeSize(5000) .compress(CompressionKind.SNAPPY) .bufferSize(1000) .rowIndexStride(0));
.fileSystem(fs) .stripeSize(stripeSize) .compress(compress) .bufferSize(bufferSize) .rowIndexStride(rowIndexStride));
OrcFile.writerOptions(conf) .inspector(inspector) .compress(CompressionKind.NONE) .stripeSize(50000) .bufferSize(100)
.inspector(inspector) .stripeSize(1000) .compress(CompressionKind.SNAPPY) .bufferSize(100)); Random rand = new Random(12);
OrcFile.writerOptions(conf) .inspector(inspector) .compress(CompressionKind.NONE) .stripeSize(50000) .bufferSize(100)
if(!MetastoreConf.getBoolVar(options.getConfiguration(), MetastoreConf.ConfVars.COMPACTOR_MINOR_STATS_COMPRESSION)) { opts.compress(CompressionKind.NONE).rowIndexStride(0);
OrcFile.WriterOptions opts = OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB); Writer writer = OrcFile.createWriter(new Path(testFilePath, "-0"), opts); writer.close(); LlapDaemonInfo.initialize("test", new Configuration()); LlapProxy.setDaemon(true); opts = OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB); writer = OrcFile.createWriter(new Path(testFilePath, "-1"), opts); writer.close(); opts = OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB); writer = OrcFile.createWriter(new Path(testFilePath, "-2"), opts); writer.close();
@Test public void testBitPack64Large() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } int size = 1080832; long[] inp = new long[size]; Random rand = new Random(1234); for (int i = 0; i < size; i++) { inp[i] = rand.nextLong(); } List<Long> input = Lists.newArrayList(Longs.asList(inp)); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB)); for (Long l : input) { writer.addRow(l); } writer.close(); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); RecordReader rows = reader.rows(); int idx = 0; while (rows.hasNext()) { Object row = rows.next(null); Assert.assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get()); } }
.fileSystem(fs) .stripeSize(stripeSize) .compress(compress) .bufferSize(bufferSize) .rowIndexStride(rowIndexStride));
private void createOrcFile(int stripSize, int numberOfRows) throws IOException { ObjectInspector inspector; synchronized (TestOrcFileStripeMergeRecordReader.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (StringIntIntIntRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(tmpPath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(stripSize) .compress(CompressionKind.ZLIB) .bufferSize(5000) .rowIndexStride(1000)); Random rand = new Random(157); for (int i = 0; i < numberOfRows; i++) { writer.addRow(new StringIntIntIntRow( Integer.toBinaryString(i), rand.nextInt(), rand.nextInt(), rand.nextInt() )); } writer.close(); }
static WriterOptions createOrcWriterOptions(ObjectInspector sourceOi, Configuration conf, CacheWriter cacheWriter, int allocSize) throws IOException { return OrcFile.writerOptions(conf).stripeSize(Long.MAX_VALUE).blockSize(Long.MAX_VALUE) .rowIndexStride(Integer.MAX_VALUE) // For now, do not limit this - one RG per split .blockPadding(false).compress(CompressionKind.NONE).version(Version.CURRENT) .encodingStrategy(EncodingStrategy.SPEED).bloomFilterColumns(null).inspector(sourceOi) .physicalWriter(cacheWriter).memory(MEMORY_MANAGER).bufferSize(allocSize); }