@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
@Override protected Codec createInstance() { return new BZip2Codec(); } }
public void addFile(String source, String destination, Configuration paramConfiguration) throws IOException, URISyntaxException { FileSystem localFileSystem = FileSystem.get(paramConfiguration); String str1 = paramString1.substring(source.lastIndexOf('/') + 1, source.length()); if (destination.charAt(destination.length() - 1) != '/') { destination = destination + "/" + str1; } else { destination = destination + str1; } BZip2Codec localBZip2Codec = new BZip2Codec(); String str2 = localBZip2Codec.getDefaultExtension(); Path localPath = new Path(paramString2 + str2); CompressionOutputStream localCompressionOutputStream = localBZip2Codec.createOutputStream(localFileSystem.create(localPath)); IOUtils.copyBytes(localFileSystem.open(new Path(paramString1)), localCompressionOutputStream, 4096, true); }
JavaPairRDD<String, PortableDataStream> imageByteRDD = jsc.binaryFiles(SOURCE_PATH); if(!imageByteRDD.isEmpty()) imageByteRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String,PortableDataStream>>>() { @Override public void call( Iterator<Tuple2<String, PortableDataStream>> arg0) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", HDFS_PATH); while(arg0.hasNext()){ Tuple2<String,PortableDataStream>fileTuple = arg0.next(); Text key = new Text(fileTuple._1()); String fileName = key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-1].split(DOT_REGEX)[0]; String fileExtension = fileName.split(DOT_REGEX)[fileName.split(DOT_REGEX).length-1]; BytesWritable value = new BytesWritable( fileTuple._2().toArray()); SequenceFile.Writer writer = SequenceFile.createWriter( conf, SequenceFile.Writer.file(new Path(DEST_PATH + fileName + SEP_KEY + getCurrentTimeStamp()+DOT+fileExtension)), SequenceFile.Writer.compression(SequenceFile.CompressionType.RECORD, new BZip2Codec()), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(BytesWritable.class)); key = new Text(key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-2] + SEP_KEY + fileName + SEP_KEY + fileExtension); writer.append(key, value); IOUtils.closeStream(writer); } } });
JavaPairRDD<String, PortableDataStream> imageByteRDD = jsc.binaryFiles(SOURCE_PATH); if(!imageByteRDD.isEmpty()) imageByteRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String,PortableDataStream>>>() { @Override public void call( Iterator<Tuple2<String, PortableDataStream>> arg0) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", HDFS_PATH); while(arg0.hasNext()){ Tuple2<String,PortableDataStream>fileTuple = arg0.next(); Text key = new Text(fileTuple._1()); String fileName = key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-1].split(DOT_REGEX)[0]; String fileExtension = fileName.split(DOT_REGEX)[fileName.split(DOT_REGEX).length-1]; BytesWritable value = new BytesWritable( fileTuple._2().toArray()); SequenceFile.Writer writer = SequenceFile.createWriter( conf, SequenceFile.Writer.file(new Path(DEST_PATH + fileName + SEP_KEY + getCurrentTimeStamp()+DOT+fileExtension)), SequenceFile.Writer.compression(SequenceFile.CompressionType.RECORD, new BZip2Codec()), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(BytesWritable.class)); key = new Text(key.toString().split(SEP_PATH)[key.toString().split(SEP_PATH).length-2] + SEP_KEY + fileName + SEP_KEY + fileExtension); writer.append(key, value); IOUtils.closeStream(writer); } } });