@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override public int hashCode() { return getName().hashCode(); }
@Override public ByteBuffer compress(ByteBuffer uncompressedData) throws IOException { ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining()); try (BZip2CompressorOutputStream outputStream = new BZip2CompressorOutputStream(baos)) { outputStream.write(uncompressedData.array(), computeOffset(uncompressedData), uncompressedData.remaining()); } return ByteBuffer.wrap(baos.toByteArray()); }
public void addFile(String source, String destination, Configuration paramConfiguration) throws IOException, URISyntaxException { FileSystem localFileSystem = FileSystem.get(paramConfiguration); String str1 = paramString1.substring(source.lastIndexOf('/') + 1, source.length()); if (destination.charAt(destination.length() - 1) != '/') { destination = destination + "/" + str1; } else { destination = destination + str1; } BZip2Codec localBZip2Codec = new BZip2Codec(); String str2 = localBZip2Codec.getDefaultExtension(); Path localPath = new Path(paramString2 + str2); CompressionOutputStream localCompressionOutputStream = localBZip2Codec.createOutputStream(localFileSystem.create(localPath)); IOUtils.copyBytes(localFileSystem.open(new Path(paramString1)), localCompressionOutputStream, 4096, true); }
@Override public ByteBuffer compress(ByteBuffer uncompressedData) throws IOException { ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining()); BZip2CompressorOutputStream outputStream = new BZip2CompressorOutputStream(baos); try { outputStream.write(uncompressedData.array(), uncompressedData.position(), uncompressedData.remaining()); } finally { outputStream.close(); } ByteBuffer result = ByteBuffer.wrap(baos.toByteArray()); return result; }
@Override public ByteBuffer decompress(ByteBuffer compressedData) throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(compressedData.array(), computeOffset(compressedData), compressedData.remaining()); try(BZip2CompressorInputStream inputStream = new BZip2CompressorInputStream(bais)) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int readCount = -1; while ((readCount = inputStream.read(buffer, compressedData.position(), buffer.length)) > 0) { baos.write(buffer, 0, readCount); } return ByteBuffer.wrap(baos.toByteArray()); } }
@Override public ByteBuffer compress(ByteBuffer uncompressedData) throws IOException { ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining()); BZip2CompressorOutputStream outputStream = new BZip2CompressorOutputStream(baos); try { outputStream.write(uncompressedData.array(), uncompressedData.position(), uncompressedData.remaining()); } finally { outputStream.close(); } ByteBuffer result = ByteBuffer.wrap(baos.toByteArray()); return result; }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override public int hashCode() { return getName().hashCode(); }
@Override public ByteBuffer compress(ByteBuffer uncompressedData) throws IOException { ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining()); BZip2CompressorOutputStream outputStream = new BZip2CompressorOutputStream(baos); try { outputStream.write(uncompressedData.array(), uncompressedData.position(), uncompressedData.remaining()); } finally { outputStream.close(); } ByteBuffer result = ByteBuffer.wrap(baos.toByteArray()); return result; }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override public int hashCode() { return getName().hashCode(); }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override public int hashCode() { return getName().hashCode(); }
JavaPairRDD<String, PortableDataStream> imageByteRDD = jsc.binaryFiles(SOURCE_PATH); if(!imageByteRDD.isEmpty()) imageByteRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String,PortableDataStream>>>() { @Override public void call( Iterator<Tuple2<String, PortableDataStream>> arg0) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", HDFS_PATH); while(arg0.hasNext()){ Tuple2<String,PortableDataStream>fileTuple = arg0.next(); Text key = new Text(fileTuple._1()); String fileName = key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-1].split(DOT_REGEX)[0]; String fileExtension = fileName.split(DOT_REGEX)[fileName.split(DOT_REGEX).length-1]; BytesWritable value = new BytesWritable( fileTuple._2().toArray()); SequenceFile.Writer writer = SequenceFile.createWriter( conf, SequenceFile.Writer.file(new Path(DEST_PATH + fileName + SEP_KEY + getCurrentTimeStamp()+DOT+fileExtension)), SequenceFile.Writer.compression(SequenceFile.CompressionType.RECORD, new BZip2Codec()), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(BytesWritable.class)); key = new Text(key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-2] + SEP_KEY + fileName + SEP_KEY + fileExtension); writer.append(key, value); IOUtils.closeStream(writer); } } });
JavaPairRDD<String, PortableDataStream> imageByteRDD = jsc.binaryFiles(SOURCE_PATH); if(!imageByteRDD.isEmpty()) imageByteRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String,PortableDataStream>>>() { @Override public void call( Iterator<Tuple2<String, PortableDataStream>> arg0) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", HDFS_PATH); while(arg0.hasNext()){ Tuple2<String,PortableDataStream>fileTuple = arg0.next(); Text key = new Text(fileTuple._1()); String fileName = key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-1].split(DOT_REGEX)[0]; String fileExtension = fileName.split(DOT_REGEX)[fileName.split(DOT_REGEX).length-1]; BytesWritable value = new BytesWritable( fileTuple._2().toArray()); SequenceFile.Writer writer = SequenceFile.createWriter( conf, SequenceFile.Writer.file(new Path(DEST_PATH + fileName + SEP_KEY + getCurrentTimeStamp()+DOT+fileExtension)), SequenceFile.Writer.compression(SequenceFile.CompressionType.RECORD, new BZip2Codec()), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(BytesWritable.class)); key = new Text(key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-2] + SEP_KEY + fileName + SEP_KEY + fileExtension); writer.append(key, value); IOUtils.closeStream(writer); } } });