/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } if (decompressor != null && !decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { updateLeaseCount(decompressorCounts, decompressor, 1); } return decompressor; }
public Object makeObject() throws Exception { return codec.createDecompressor(); } }, Integer.MAX_VALUE);
public static Decompressor getDecompressor(CompressionCodec codec) { return codec.createDecompressor(); }
public Object makeObject() throws Exception { return codec.createDecompressor(); } }, Integer.MAX_VALUE);
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor"); } else { LOG.debug("Got recycled decompressor"); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec * the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec .getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } if (decompressor != null && !decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { updateLeaseCount(decompressorCounts, decompressor, 1); } return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } updateLeaseCount(decompressorCounts, decompressor, 1); return decompressor; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } updateLeaseCount(decompressorCounts, decompressor, 1); return decompressor; }
private static void requireNativeZlib() { Configuration conf = new Configuration(); if (!ZlibFactory.isNativeZlibLoaded(conf)) { throw new RuntimeException("native zlib is not loaded"); } CompressionCodecFactory factory = new CompressionCodecFactory(conf); CompressionCodec codec = factory.getCodecByClassName(GzipCodec.class.getName()); if (codec == null) { throw new RuntimeException("failed to load GzipCodec"); } org.apache.hadoop.io.compress.Decompressor decompressor = codec.createDecompressor(); if (!(decompressor instanceof ZlibDecompressor)) { throw new RuntimeException("wrong gzip decompressor: " + decompressor.getClass().getName()); } }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } if (decompressor != null && !decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { updateLeaseCount(decompressorCounts, decompressor, 1); } return decompressor; }
@Test public void testGzipCompatibility() throws IOException { Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); LOG.info("seed: " + seed); DataOutputBuffer dflbuf = new DataOutputBuffer(); GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); byte[] b = new byte[r.nextInt(128 * 1024 + 1)]; r.nextBytes(b); gzout.write(b); gzout.close(); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.reset(dflbuf.getData(), dflbuf.getLength()); Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false); CompressionCodec codec = ReflectionUtils.newInstance(GzipCodec.class, conf); Decompressor decom = codec.createDecompressor(); assertNotNull(decom); assertEquals(BuiltInGzipDecompressor.class, decom.getClass()); InputStream gzin = codec.createInputStream(gzbuf, decom); dflbuf.reset(); IOUtils.copyBytes(gzin, dflbuf, 4096); final byte[] dflchk = Arrays.copyOf(dflbuf.getData(), dflbuf.getLength()); assertArrayEquals(b, dflchk); }
@Test public void testGzipCompatibility() throws IOException { Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); LOG.info("seed: " + seed); DataOutputBuffer dflbuf = new DataOutputBuffer(); GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); byte[] b = new byte[r.nextInt(128 * 1024 + 1)]; r.nextBytes(b); gzout.write(b); gzout.close(); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.reset(dflbuf.getData(), dflbuf.getLength()); Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false); CompressionCodec codec = ReflectionUtils.newInstance(GzipCodec.class, conf); Decompressor decom = codec.createDecompressor(); assertNotNull(decom); assertEquals(BuiltInGzipDecompressor.class, decom.getClass()); InputStream gzin = codec.createInputStream(gzbuf, decom); dflbuf.reset(); IOUtils.copyBytes(gzin, dflbuf, 4096); final byte[] dflchk = Arrays.copyOf(dflbuf.getData(), dflbuf.getLength()); assertArrayEquals(b, dflchk); }