private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
boolean mergePhaseFinished = false; if (inMemoryMapOutputs.size() > 0) { TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, memDiskSegments,
boolean mergePhaseFinished = false; if (inMemoryMapOutputs.size() > 0) { TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, memDiskSegments,
boolean mergePhaseFinished = false; if (inMemoryMapOutputs.size() > 0) { TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, memDiskSegments,
boolean mergePhaseFinished = false; if (inMemoryMapOutputs.size() > 0) { TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, memDiskSegments,