/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
do { factor = getPassFactor(passNo, numSegments); List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>(); getSegmentDescriptors(numSegmentsToConsider); for (int i = 0; i < mStream.length; i++) { if (mStream[i].nextRawKey()) { segmentsConsidered++; updateProgress(mStream[i].in.getPosition()); initialize(segmentsToMerge.size()); clear(); for (int i = 0; i < segmentsToMerge.size(); i++) { put(segmentsToMerge.get(i)); this.close();
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }