private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
@Override public boolean pushRecord() throws HiveException { execContext.resetRow(); try { if (reader.next()) { Object value; try { value = reader.getCurrentValue(); } catch (IOException e) { closeReader(); throw new HiveException(e); } return processRow(value); } } catch (IOException e) { closeReader(); throw new HiveException(e); } return false; }
@Override void run() throws Exception { startAbortChecks(); while (sources[position].pushRecord()) { addRowAndMaybeCheckAbort(); } }
@Override public boolean pushRecord() throws HiveException { execContext.resetRow(); try { if (reader.next()) { Object value; try { value = reader.getCurrentValue(); } catch (IOException e) { throw new HiveException(e); } return processRow(value); } } catch (IOException e) { throw new HiveException(e); } return false; }
private boolean processRow(Object value) { try { if (mapOp.getDone()) { return false; // done } else { // Since there is no concept of a group, we don't invoke // startGroup/endGroup for a mapper mapOp.process((Writable) value); } } catch (Throwable e) { if (e instanceof OutOfMemoryError) { // Don't create a new object if we are already out of memory throw (OutOfMemoryError) e; } else { LOG.error(StringUtils.stringifyException(e)); closeReader(); throw new RuntimeException(e); } } return true; // give me more }
private boolean processRow(Object value) { try { if (mapOp.getDone()) { return false; // done } else { // Since there is no concept of a group, we don't invoke // startGroup/endGroup for a mapper mapOp.process((Writable) value); } } catch (Throwable e) { if (e instanceof OutOfMemoryError) { // Don't create a new object if we are already out of memory throw (OutOfMemoryError) e; } else { LOG.error(StringUtils.stringifyException(e)); closeReader(); throw new RuntimeException(e); } } return true; // give me more }
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
@Override public boolean pushRecord() throws HiveException { execContext.resetRow(); try { if (reader.next()) { Object value; try { value = reader.getCurrentValue(); } catch (IOException e) { closeReader(); throw new HiveException(e); } return processRow(value); } } catch (IOException e) { closeReader(); throw new HiveException(e); } return false; }
@Override void run() throws Exception { startAbortChecks(); while (sources[position].pushRecord()) { addRowAndMaybeCheckAbort(); } }
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (MapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } ((TezContext) MapredContext.get()).setRecordSources(sources); }
@Override void run() throws Exception { while (sources[position].pushRecord()) {} }