private FSDataOutputStream getOutputStream() throws IOException { if (this.outputStream == null) { this.outputStream = this.fs.create(this.checkpointFile, false, this.buf.length, this.replication, this.fs.getDefaultBlockSize()); } return this.outputStream; }
private FSDataOutputStream getOutputStream() throws IOException { if (this.outputStream == null) { this.outputStream = this.fs.create(this.checkpointFile, false, this.buf.length, this.replication, this.fs.getDefaultBlockSize()); } return this.outputStream; }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedInput = new BlockBasedOutput(this.stream, (int) blockSize); this.dataOutputStream = new DataOutputStream(this.blockBasedInput); }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedInput = new BlockBasedOutput(this.stream, (int) blockSize); this.dataOutputStream = new DataOutputStream(this.blockBasedInput); }
@Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { List<FileStatus> files = this.getFiles(); final FileSystem fs = this.filePath.getFileSystem(); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? fs.getDefaultBlockSize() : this.blockSize; final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits); for (FileStatus file : files) { long splitSize = blockSize; for (long pos = 0, length = file.getLen(); pos < length; pos += splitSize) { long remainingLength = Math.min(pos + splitSize, length) - pos; // get the block locations and make sure they are in order with respect to their offset final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength); Arrays.sort(blocks); inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength, blocks[0].getHosts())); } } if (inputSplits.size() < minNumSplits) { LOG.warn(String.format( "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...", blockSize, this.filePath, minNumSplits)); FileStatus last = files.get(files.size() - 1); final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen()); for (int index = files.size(); index < minNumSplits; index++) { inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts())); } } return inputSplits.toArray(new FileInputSplit[0]); }
@Override public void open(final int taskNumber) throws IOException { super.open(taskNumber); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedInput = new BlockBasedOutput(this.stream, (int) blockSize); this.dataOutputStream = new DataOutputViewStream(this.blockBasedInput); }
@Override public FileInputSplit[] createInputSplits(final int minNumSplits) throws IOException { final List<FileStatus> files = this.getFiles(); final FileSystem fs = this.filePath.getFileSystem(); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? fs.getDefaultBlockSize() : this.blockSize; final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits); for (final FileStatus file : files) { final long splitSize = blockSize; for (long pos = 0, length = file.getLen(); pos < length; pos += splitSize) { final long remainingLength = Math.min(pos + splitSize, length) - pos; // get the block locations and make sure they are in order with respect to their offset final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength); Arrays.sort(blocks); inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength, blocks[0].getHosts())); } } if (inputSplits.size() < minNumSplits) { LOG.warn(String.format( "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...", blockSize, this.filePath, minNumSplits)); final FileStatus last = files.get(files.size() - 1); final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen()); for (int index = files.size(); index < minNumSplits; index++) inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts())); } return inputSplits.toArray(new FileInputSplit[0]); }
@Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { List<FileStatus> files = this.getFiles(); final FileSystem fs = this.filePath.getFileSystem(); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? fs.getDefaultBlockSize() : this.blockSize; final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits); for (FileStatus file : files) { long splitSize = blockSize; for (long pos = 0, length = file.getLen(); pos < length; pos += splitSize) { long remainingLength = Math.min(pos + splitSize, length) - pos; // get the block locations and make sure they are in order with respect to their offset final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength); Arrays.sort(blocks); inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength, blocks[0].getHosts())); } } if (inputSplits.size() < minNumSplits) { LOG.warn(String.format( "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...", blockSize, this.filePath, minNumSplits)); FileStatus last = files.get(files.size() - 1); final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen()); for (int index = files.size(); index < minNumSplits; index++) { inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts())); } } return inputSplits.toArray(new FileInputSplit[0]); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputStream(this.blockBasedInput); this.readRecords = 0; }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputStream(this.blockBasedInput); this.readRecords = 0; }
@Override public void open(final FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); final DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputViewStream(this.blockBasedInput); this.readRecords = 0; }