public boolean nextRawKey(DataInputBuffer key) throws IOException { try { if (!positionToNextRecord(memDataIn)) { return false; } // Setup the key int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); key.reset(data, pos, currentKeyLength); // Position for the next value long skipped = memDataIn.skip(currentKeyLength); if (skipped != currentKeyLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " + currentKeyLength); } // Record the byte bytesRead += currentKeyLength; return true; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
public void nextRawValue(DataInputBuffer value) throws IOException { try { int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); value.reset(data, pos, currentValueLength); // Position for the next record long skipped = memDataIn.skip(currentValueLength); if (skipped != currentValueLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past value of length: " + currentValueLength); } // Record the byte bytesRead += currentValueLength; ++recNo; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public void nextRawValue(DataInputBuffer value) throws IOException { try { int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); value.reset(data, pos, currentValueLength); // Position for the next record long skipped = memDataIn.skip(currentValueLength); if (skipped != currentValueLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past value of length: " + currentValueLength); } // Record the byte bytesRead += currentValueLength; ++recNo; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
private long createInMemorySegments(List<MapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (MapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { MapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManager.this, mo.getMapId(), data, 0, (int)size); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public boolean nextRawKey(DataInputBuffer key) throws IOException { try { if (!positionToNextRecord(memDataIn)) { return false; } // Setup the key int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); key.reset(data, pos, currentKeyLength); // Position for the next value long skipped = memDataIn.skip(currentKeyLength); if (skipped != currentKeyLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " + currentKeyLength); } // Record the byte bytesRead += currentKeyLength; return true; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
public void nextRawValue(DataInputBuffer value) throws IOException { try { int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); value.reset(data, pos, currentValueLength); // Position for the next record long skipped = memDataIn.skip(currentValueLength); if (skipped != currentValueLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past value of length: " + currentValueLength); } // Record the byte bytesRead += currentValueLength; ++recNo; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public boolean nextRawKey(DataInputBuffer key) throws IOException { try { if (!positionToNextRecord(memDataIn)) { return false; } // Setup the key int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); key.reset(data, pos, currentKeyLength); // Position for the next value long skipped = memDataIn.skip(currentKeyLength); if (skipped != currentKeyLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " + currentKeyLength); } // Record the byte bytesRead += currentKeyLength; return true; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
public void nextRawValue(DataInputBuffer value) throws IOException { try { int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); value.reset(data, pos, currentValueLength); // Position for the next record long skipped = memDataIn.skip(currentValueLength); if (skipped != currentValueLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past value of length: " + currentValueLength); } // Record the byte bytesRead += currentValueLength; ++recNo; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public boolean nextRawKey(DataInputBuffer key) throws IOException { try { if (!positionToNextRecord(memDataIn)) { return false; } // Setup the key int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); key.reset(data, pos, currentKeyLength); // Position for the next value long skipped = memDataIn.skip(currentKeyLength); if (skipped != currentKeyLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " + currentKeyLength); } // Record the byte bytesRead += currentKeyLength; return true; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
public void nextRawValue(DataInputBuffer value) throws IOException { try { int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); value.reset(data, pos, currentValueLength); // Position for the next record long skipped = memDataIn.skip(currentValueLength); if (skipped != currentValueLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past value of length: " + currentValueLength); } // Record the byte bytesRead += currentValueLength; ++recNo; } catch (IOException ioe) { dumpOnError(); throw ioe; } }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public boolean nextRawKey(DataInputBuffer key) throws IOException { try { if (!positionToNextRecord(memDataIn)) { return false; } // Setup the key int pos = memDataIn.getPosition(); byte[] data = memDataIn.getData(); key.reset(data, pos, currentKeyLength); // Position for the next value long skipped = memDataIn.skip(currentKeyLength); if (skipped != currentKeyLength) { throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " + currentKeyLength); } // Record the byte bytesRead += currentKeyLength; return true; } catch (IOException ioe) { dumpOnError(); throw ioe; } }