@Override public void merge(List<InMemoryMapOutput<Text, Text>> inputs) throws IOException { synchronized (this) { numMerges.incrementAndGet(); for (InMemoryMapOutput<Text, Text> input : inputs) { manager.unreserve(input.getSize()); } } try { mergeStart.await(); mergeComplete.await(); } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } } }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
public synchronized void closeInMemoryFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMapOutputs.add(mapOutput); LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize() + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size() + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory); commitMemory+= mapOutput.getSize(); // Can hang if mergeThreshold is really low. if (commitMemory >= mergeThreshold) { LOG.info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold=" + mergeThreshold + ". Current usedMemory=" + usedMemory); inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs); inMemoryMergedMapOutputs.clear(); inMemoryMerger.startMerge(inMemoryMapOutputs); commitMemory = 0L; // Reset commitMemory. } if (memToMemMerger != null) { if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) { memToMemMerger.startMerge(inMemoryMapOutputs); } } }
public synchronized void closeInMemoryFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMapOutputs.add(mapOutput); LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize() + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size() + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory); commitMemory+= mapOutput.getSize(); // Can hang if mergeThreshold is really low. if (commitMemory >= mergeThreshold) { LOG.info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold=" + mergeThreshold + ". Current usedMemory=" + usedMemory); inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs); inMemoryMergedMapOutputs.clear(); inMemoryMerger.startMerge(inMemoryMapOutputs); commitMemory = 0L; // Reset commitMemory. } if (memToMemMerger != null) { if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) { memToMemMerger.startMerge(inMemoryMapOutputs); } } }
public synchronized void closeInMemoryFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMapOutputs.add(mapOutput); LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize() + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size() + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory); commitMemory+= mapOutput.getSize(); // Can hang if mergeThreshold is really low. if (commitMemory >= mergeThreshold) { LOG.info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold=" + mergeThreshold + ". Current usedMemory=" + usedMemory); inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs); inMemoryMergedMapOutputs.clear(); inMemoryMerger.startMerge(inMemoryMapOutputs); commitMemory = 0L; // Reset commitMemory. } if (memToMemMerger != null) { if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) { memToMemMerger.startMerge(inMemoryMapOutputs); } } }
public synchronized void closeInMemoryFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMapOutputs.add(mapOutput); LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize() + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size() + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory); commitMemory+= mapOutput.getSize(); // Can hang if mergeThreshold is really low. if (commitMemory >= mergeThreshold) { LOG.info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold=" + mergeThreshold + ". Current usedMemory=" + usedMemory); inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs); inMemoryMergedMapOutputs.clear(); inMemoryMerger.startMerge(inMemoryMapOutputs); commitMemory = 0L; // Reset commitMemory. } if (memToMemMerger != null) { if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) { memToMemMerger.startMerge(inMemoryMapOutputs); } } }