@Override public Reader getReader() throws Exception { return new KeyValuesInputMerger(getInputs()); }
@Override public Iterable<Object> getCurrentValues() throws IOException { kvsIterable.init(nextKVReaders); return kvsIterable; }
public void init(List<KeyValuesReader> readerList) { currentIterator.init(readerList); } }
public KeyValuesInputMerger(List<? extends Input> shuffleInputs) throws Exception { //get KeyValuesReaders from the LogicalInput and add them to priority queue int initialCapacity = shuffleInputs.size(); kvsIterable = new KeyValuesIterable(initialCapacity); pQueue = new PriorityQueue<KeyValuesReader>(initialCapacity, new KVReaderComparator()); for(Input input : shuffleInputs){ addToQueue((KeyValuesReader)input.getReader()); } }
@Override public void setConstituentInputIsReady(Input input) { synchronized (this) { readyInputs.put(input, Boolean.TRUE); } if (readyInputs.size() == getInputs().size()) { informInputReady(); } } }
public KeyValueInputMerger(List<KeyValueReader> multiMRInputs, Deserializer deserializer, ObjectInspector[] inputObjInspectors, List<String> sortCols) throws Exception { //get KeyValuesReaders from the LogicalInput and add them to priority queue int initialCapacity = multiMRInputs.size(); pQueue = new PriorityQueue<KeyValueReader>(initialCapacity, new KVReaderComparator()); this.inputObjInspectors = inputObjInspectors; this.deserializer = deserializer; fieldOIs = new ArrayList<ObjectInspector>(); structFields = new ArrayList<StructField>(); StructObjectInspector structOI = (StructObjectInspector) inputObjInspectors[0]; for (String field : sortCols) { StructField sf = structOI.getStructFieldRef(field); structFields.add(sf); ObjectInspector stdOI = ObjectInspectorUtils.getStandardObjectInspector(sf.getFieldObjectInspector()); fieldOIs.add(stdOI); } l4j.info("Initialized the priority queue with multi mr inputs: " + multiMRInputs.size()); for (KeyValueReader input : multiMRInputs) { addToQueue(input); } }
KeyValuesIterable(int size) { currentIterator = new KeyValuesIterator(size); }
/** * @return true if there are more key-values and advances to next key-values * @throws IOException */ @Override public boolean next() throws IOException { //add the previous nextKVReader back to queue if(nextKVReader != null){ addToQueue(nextKVReader); } //get the new nextKVReader with lowest key nextKVReader = pQueue.poll(); return nextKVReader != null; }
@SuppressWarnings("deprecation") private KeyValueReader getKeyValueReader(Collection<KeyValueReader> keyValueReaders, AbstractMapOperator mapOp) throws Exception { List<KeyValueReader> kvReaderList = new ArrayList<KeyValueReader>(keyValueReaders); // this sets up the map operator contexts correctly mapOp.initializeContexts(); Deserializer deserializer = mapOp.getCurrentDeserializer(); // deserializer is null in case of VectorMapOperator KeyValueReader reader = new KeyValueInputMerger(kvReaderList, deserializer, new ObjectInspector[] { deserializer == null ? null : deserializer.getObjectInspector() }, mapOp .getConf() .getSortCols()); return reader; }
/** * @return true if there are more key-values and advances to next key-values * @throws IOException */ @Override public boolean next() throws IOException { //add the previous nextKVReader back to queue if (!nextKVReaders.isEmpty()) { for (KeyValuesReader kvReader : nextKVReaders) { addToQueue(kvReader); } nextKVReaders.clear(); } KeyValuesReader nextKVReader = null; //get the new nextKVReader with lowest key nextKVReader = pQueue.poll(); if (nextKVReader != null) { nextKVReaders.add(nextKVReader); } while (pQueue.peek() != null) { KeyValuesReader equalValueKVReader = pQueue.poll(); if (pQueue.comparator().compare(nextKVReader, equalValueKVReader) == 0) { nextKVReaders.add(equalValueKVReader); } else { pQueue.add(equalValueKVReader); break; } } return !(nextKVReaders.isEmpty()); }
private void closeReader() { if (!(reader instanceof MRReader)) { LOG.warn("Cannot close " + (reader == null ? null : reader.getClass())); return; } if (reader instanceof KeyValueInputMerger) { // cleanup KeyValueInputMerger kvMerger = (KeyValueInputMerger) reader; kvMerger.clean(); } LOG.info("Closing MRReader on error"); MRReader mrReader = (MRReader)reader; try { mrReader.close(); } catch (IOException ex) { LOG.error("Failed to close the reader; ignoring", ex); } }
void init(JobConf jconf, AbstractMapOperator mapOp, KeyValueReader reader) throws IOException { execContext = mapOp.getExecContext(); this.mapOp = mapOp; if (reader instanceof KeyValueInputMerger) { KeyValueInputMerger kvMerger = (KeyValueInputMerger) reader; kvMerger.setIOCxt(execContext.getIoCxt()); } this.reader = reader; }
public KeyValuesInputMerger(List<? extends Input> shuffleInputs) throws Exception { //get KeyValuesReaders from the LogicalInput and add them to priority queue int initialCapacity = shuffleInputs.size(); kvsIterable = new KeyValuesIterable(initialCapacity); pQueue = new PriorityQueue<KeyValuesReader>(initialCapacity, new KVReaderComparator()); for(Input input : shuffleInputs){ addToQueue((KeyValuesReader)input.getReader()); } }
@Override public Reader getReader() throws Exception { return new KeyValuesInputMerger(getInputs()); }
@Override public void setConstituentInputIsReady(Input input) { synchronized (this) { readyInputs.put(input, Boolean.TRUE); } if (readyInputs.size() == getInputs().size()) { informInputReady(); } } }
@Override public Iterable<Object> getCurrentValues() throws IOException { kvsIterable.init(nextKVReaders); return kvsIterable; }
public void init(List<KeyValuesReader> readerList) { currentIterator.init(readerList); } }
KeyValuesIterable(int size) { currentIterator = new KeyValuesIterator(size); }
/** * @return true if there are more key-values and advances to next key-values * @throws IOException */ @Override public boolean next() throws IOException { //add the previous nextKVReader back to queue if(nextKVReader != null){ addToQueue(nextKVReader); } //get the new nextKVReader with lowest key nextKVReader = pQueue.poll(); if (nextKVReader == null) return false; if (nextKVReader != prevKVReader) { prevKVReader = nextKVReader; // update path in IOContext ioCxt.setInputPath(kvReaderPathMap.get(nextKVReader)); } return true; }
@SuppressWarnings("deprecation") private KeyValueReader getKeyValueReader(Collection<KeyValueReader> keyValueReaders, AbstractMapOperator mapOp) throws Exception { List<KeyValueReader> kvReaderList = new ArrayList<KeyValueReader>(keyValueReaders); // this sets up the map operator contexts correctly mapOp.initializeContexts(); Deserializer deserializer = mapOp.getCurrentDeserializer(); // deserializer is null in case of VectorMapOperator KeyValueReader reader = new KeyValueInputMerger(kvReaderList, deserializer, new ObjectInspector[] { deserializer == null ? null : deserializer.getObjectInspector() }, mapOp .getConf() .getSortCols()); return reader; }