private CodecFactory getCompressionCodec(Map<String, String> conf) { if (getBoolean(conf, CONF_COMPRESS, false)) { int deflateLevel = getInt(conf, CONF_DEFLATE_LEVEL, CodecFactory.DEFAULT_DEFLATE_LEVEL); int xzLevel = getInt(conf, CONF_XZ_LEVEL, CodecFactory.DEFAULT_XZ_LEVEL); String outputCodec = conf.get(CONF_COMPRESS_CODEC); if (DataFileConstants.DEFLATE_CODEC.equals(outputCodec)) { return CodecFactory.deflateCodec(deflateLevel); } else if (DataFileConstants.XZ_CODEC.equals(outputCodec)) { return CodecFactory.xzCodec(xzLevel); } else { return CodecFactory.fromString(outputCodec); } } return CodecFactory.nullCodec(); }
return CodecFactory.nullCodec();
Codec resolveCodec() { String codecStr = getMetaString(DataFileConstants.CODEC); if (codecStr != null) { return CodecFactory.fromString(codecStr).createInstance(); } else { return CodecFactory.nullCodec().createInstance(); } }
@Test public void testWithNullCodec() throws IOException { Configuration conf = new Configuration(); conf.setInt(SYNC_INTERVAL_KEY, TEST_SYNC_INTERVAL); testGetRecordWriter(conf, CodecFactory.nullCodec(), TEST_SYNC_INTERVAL); }
Codec resolveCodec() { String codecStr = getMetaString(DataFileConstants.CODEC); if (codecStr != null) { return CodecFactory.fromString(codecStr).createInstance(); } else { return CodecFactory.nullCodec().createInstance(); } }
private CodecFactory getCodec(File output) throws Exception { try (DataFileStream<GenericRecord> reader = new DataFileStream<>( new FileInputStream(output), new GenericDatumReader<>())) { String codec = reader.getMetaString(DataFileConstants.CODEC); return codec == null ? CodecFactory.nullCodec() : CodecFactory.fromString(codec); } }
/** Open a writer appending to an existing file. * <strong>Since 1.9.0 this method does not close in.</strong> * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<>(in, new GenericDatumReader<>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } init(out); return this; }
/** Open a writer appending to an existing file. * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<D>(in, new GenericDatumReader<D>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } reader.close(); init(out); return this; }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
public static CodecFactory getCodecFactory(String property) { CodecType type = CodecType.valueOf(property); switch (type) { case BZIP2: return CodecFactory.bzip2Codec(); case DEFLATE: return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL); case LZO: return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL); case SNAPPY: return CodecFactory.snappyCodec(); case NONE: default: return CodecFactory.nullCodec(); } }
private CodecFactory getCodecFactory(String property) { CodecType type = CodecType.valueOf(property); switch (type) { case BZIP2: return CodecFactory.bzip2Codec(); case DEFLATE: return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL); case LZO: return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL); case SNAPPY: return CodecFactory.snappyCodec(); case NONE: default: return CodecFactory.nullCodec(); } }
protected CodecFactory getCodecFactory(String property) { CodecType type = CodecType.valueOf(property); switch (type) { case BZIP2: return CodecFactory.bzip2Codec(); case DEFLATE: return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL); case NONE: return CodecFactory.nullCodec(); case LZO: return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL); case SNAPPY: default: return CodecFactory.snappyCodec(); } }
@Parameters public static List<Object[]> codecs() { List<Object[]> r = new ArrayList<>(); r.add(new Object[] { null }); r.add(new Object[] { CodecFactory.deflateCodec(0) }); r.add(new Object[] { CodecFactory.deflateCodec(1) }); r.add(new Object[] { CodecFactory.deflateCodec(9) }); r.add(new Object[] { CodecFactory.nullCodec() }); r.add(new Object[] { CodecFactory.snappyCodec() }); r.add(new Object[] { CodecFactory.xzCodec(0) }); r.add(new Object[] { CodecFactory.xzCodec(1) }); r.add(new Object[] { CodecFactory.xzCodec(6) }); r.add(new Object[] { CodecFactory.zstandardCodec() }); return r; }
@Parameters public static List<Object[]> codecs() { List<Object[]> r = new ArrayList<>(); r.add(new Object[] { null , null, false}); r.add(new Object[] { null , null, true}); r.add(new Object[] { CodecFactory.deflateCodec(1), CodecFactory.deflateCodec(6), false }); r.add(new Object[] { CodecFactory.deflateCodec(1), CodecFactory.deflateCodec(6), true }); r.add(new Object[] { CodecFactory.deflateCodec(3), CodecFactory.nullCodec(), false }); r.add(new Object[] { CodecFactory.nullCodec(), CodecFactory.deflateCodec(6), false }); r.add(new Object[] { CodecFactory.xzCodec(1), CodecFactory.xzCodec(2), false }); r.add(new Object[] { CodecFactory.xzCodec(1), CodecFactory.xzCodec(2), true }); r.add(new Object[] { CodecFactory.xzCodec(2), CodecFactory.nullCodec(), false }); r.add(new Object[] { CodecFactory.nullCodec(), CodecFactory.xzCodec(2), false }); return r; }
@Test public void testDifferentCodecFail() throws Exception { Map<String, String> metadata = new HashMap<>(); metadata.put("myMetaKey", "myMetaValue"); File input1 = generateData(name.getMethodName() + "-1.avro", Type.STRING, metadata, DEFLATE); File input2 = generateData(name.getMethodName() + "-2.avro", Type.STRING, metadata, CodecFactory.nullCodec()); File output = new File(OUTPUT_DIR.getRoot(), name.getMethodName() + ".avro"); List<String> args = asList( input1.getAbsolutePath(), input2.getAbsolutePath(), output.getAbsolutePath()); int returnCode = new ConcatTool().run( System.in, System.out, System.err, args); assertEquals(3, returnCode); }
@Test public void testWrite() throws IOException { Schema writerSchema = Schema.create(Schema.Type.INT); GenericData dataModel = new ReflectData(); CodecFactory compressionCodec = CodecFactory.nullCodec(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); TaskAttemptContext context = createMock(TaskAttemptContext.class); replay(context); // Write an avro container file with two records: 1 and 2. AvroKeyRecordWriter<Integer> recordWriter = new AvroKeyRecordWriter<>( writerSchema, dataModel, compressionCodec, outputStream); recordWriter.write(new AvroKey<>(1), NullWritable.get()); recordWriter.write(new AvroKey<>(2), NullWritable.get()); recordWriter.close(context); verify(context); // Verify that the file was written as expected. InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); Schema readerSchema = Schema.create(Schema.Type.INT); DatumReader<Integer> datumReader = new SpecificDatumReader<>(readerSchema); DataFileStream<Integer> dataFileReader = new DataFileStream<>(inputStream, datumReader); assertTrue(dataFileReader.hasNext()); // Record 1. assertEquals(1, dataFileReader.next().intValue()); assertTrue(dataFileReader.hasNext()); // Record 2. assertEquals(2, dataFileReader.next().intValue()); assertFalse(dataFileReader.hasNext()); // No more records. dataFileReader.close(); }
Schema writerSchema = Schema.create(Schema.Type.INT); GenericData dataModel = new ReflectData(); CodecFactory compressionCodec = CodecFactory.nullCodec(); FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro")); TaskAttemptContext context = createMock(TaskAttemptContext.class);
new AvroKeyValueRecordWriter<>( keyConverter, valueConverter, new ReflectData(), CodecFactory.nullCodec(), outputStream);
AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass()); CodecFactory compressionCodec = CodecFactory.nullCodec(); FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));