/** Creates spill file(s) within the local dirs. */ public void insertSpillData() throws IOException { String filename = "temp_local_" + UUID.randomUUID(); OutputStream dataStream = null; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, filename)); dataStream.write(42); } finally { Closeables.close(dataStream, false); } }
/** Creates spill file(s) within the local dirs. */ public void insertSpillData() throws IOException { String filename = "temp_local_" + UUID.randomUUID(); OutputStream dataStream = null; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, filename)); dataStream.write(42); } finally { Closeables.close(dataStream, false); } }
/** * Sort-based shuffle data uses an index called "shuffle_ShuffleId_MapId_0.index" into a data file * called "shuffle_ShuffleId_MapId_0.data". This logic is from IndexShuffleBlockResolver, * and the block id format is from ShuffleDataBlockId and ShuffleIndexBlockId. */ private ManagedBuffer getSortBasedShuffleBlockData( ExecutorShuffleInfo executor, int shuffleId, int mapId, int reduceId) { File indexFile = getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.index"); try { ShuffleIndexInformation shuffleIndexInformation = shuffleIndexCache.get(indexFile); ShuffleIndexRecord shuffleIndexRecord = shuffleIndexInformation.getIndex(reduceId); return new FileSegmentManagedBuffer( conf, getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.data"), shuffleIndexRecord.getOffset(), shuffleIndexRecord.getLength()); } catch (ExecutionException e) { throw new RuntimeException("Failed to open file: " + indexFile, e); } }
/** * Sort-based shuffle data uses an index called "shuffle_ShuffleId_MapId_0.index" into a data file * called "shuffle_ShuffleId_MapId_0.data". This logic is from IndexShuffleBlockResolver, * and the block id format is from ShuffleDataBlockId and ShuffleIndexBlockId. */ private ManagedBuffer getSortBasedShuffleBlockData( ExecutorShuffleInfo executor, int shuffleId, int mapId, int reduceId) { File indexFile = getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.index"); try { ShuffleIndexInformation shuffleIndexInformation = shuffleIndexCache.get(indexFile); ShuffleIndexRecord shuffleIndexRecord = shuffleIndexInformation.getIndex(reduceId); return new FileSegmentManagedBuffer( conf, getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.data"), shuffleIndexRecord.getOffset(), shuffleIndexRecord.getLength()); } catch (ExecutionException e) { throw new RuntimeException("Failed to open file: " + indexFile, e); } }
/** * Sort-based shuffle data uses an index called "shuffle_ShuffleId_MapId_0.index" into a data file * called "shuffle_ShuffleId_MapId_0.data". This logic is from IndexShuffleBlockResolver, * and the block id format is from ShuffleDataBlockId and ShuffleIndexBlockId. */ private ManagedBuffer getSortBasedShuffleBlockData( ExecutorShuffleInfo executor, int shuffleId, int mapId, int reduceId) { File indexFile = getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.index"); try { ShuffleIndexInformation shuffleIndexInformation = shuffleIndexCache.get(indexFile); ShuffleIndexRecord shuffleIndexRecord = shuffleIndexInformation.getIndex(reduceId); return new FileSegmentManagedBuffer( conf, getFile(executor.localDirs, executor.subDirsPerLocalDir, "shuffle_" + shuffleId + "_" + mapId + "_0.data"), shuffleIndexRecord.getOffset(), shuffleIndexRecord.getLength()); } catch (ExecutionException e) { throw new RuntimeException("Failed to open file: " + indexFile, e); } }
/** Creates reducer blocks in a sort-based data format within our local dirs. */ public void insertSortShuffleData(int shuffleId, int mapId, byte[][] blocks) throws IOException { String blockId = "shuffle_" + shuffleId + "_" + mapId + "_0"; OutputStream dataStream = null; DataOutputStream indexStream = null; boolean suppressExceptionsDuringClose = true; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".data")); indexStream = new DataOutputStream(new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".index"))); long offset = 0; indexStream.writeLong(offset); for (byte[] block : blocks) { offset += block.length; dataStream.write(block); indexStream.writeLong(offset); } suppressExceptionsDuringClose = false; } finally { Closeables.close(dataStream, suppressExceptionsDuringClose); Closeables.close(indexStream, suppressExceptionsDuringClose); } }
/** Creates reducer blocks in a sort-based data format within our local dirs. */ public void insertSortShuffleData(int shuffleId, int mapId, byte[][] blocks) throws IOException { String blockId = "shuffle_" + shuffleId + "_" + mapId + "_0"; OutputStream dataStream = null; DataOutputStream indexStream = null; boolean suppressExceptionsDuringClose = true; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".data")); indexStream = new DataOutputStream(new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".index"))); long offset = 0; indexStream.writeLong(offset); for (byte[] block : blocks) { offset += block.length; dataStream.write(block); indexStream.writeLong(offset); } suppressExceptionsDuringClose = false; } finally { Closeables.close(dataStream, suppressExceptionsDuringClose); Closeables.close(indexStream, suppressExceptionsDuringClose); } }
/** Creates reducer blocks in a sort-based data format within our local dirs. */ public void insertSortShuffleData(int shuffleId, int mapId, byte[][] blocks) throws IOException { String blockId = "shuffle_" + shuffleId + "_" + mapId + "_0"; OutputStream dataStream = null; DataOutputStream indexStream = null; boolean suppressExceptionsDuringClose = true; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".data")); indexStream = new DataOutputStream(new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, blockId + ".index"))); long offset = 0; indexStream.writeLong(offset); for (byte[] block : blocks) { offset += block.length; dataStream.write(block); indexStream.writeLong(offset); } suppressExceptionsDuringClose = false; } finally { Closeables.close(dataStream, suppressExceptionsDuringClose); Closeables.close(indexStream, suppressExceptionsDuringClose); } }