@Override void run() throws Exception { startAbortChecks(); while (sources[position].pushRecord()) { addRowAndMaybeCheckAbort(); } }
private Operator<? extends OperatorDesc> getFinalOp(Operator<? extends OperatorDesc> mergeMapOp) { for (Operator<? extends OperatorDesc> childOp : mergeMapOp.getChildOperators()) { if ((childOp.getChildOperators() == null) || (childOp.getChildOperators().isEmpty())) { return childOp; } else { return getFinalOp(childOp); } } return null; }
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); super.init(mrReporter, inputs, outputs); checkAbortCondition(); checkAbortCondition(); mergeWorkList.add( (MapWork) cache.retrieve(key, checkAbortCondition(); legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); checkAbortCondition(); createOutputMap(); checkAbortCondition(); checkAbortCondition(); Operator<? extends OperatorDesc> finalOp = getFinalOp(mergeMapOp); if (finalOp instanceof TezDummyStoreOperator) { jconf.set(Utilities.INPUT_NAME, mapWork.getName()); mapOp.initialize(jconf, null); checkAbortCondition(); mapOp.setChildren(jconf); mapOp.passExecContext(execContext);
legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); createOutputMap(); mergeMapOp.setChildren(jconf); DummyStoreOperator dummyOp = getJoinParentOp(mergeMapOp); connectOps.put(mergeMapWork.getTag(), dummyOp); initializeMapRecordSources(); mapOp.initializeMapOperator(jconf); if ((mergeMapOpList != null) && mergeMapOpList.isEmpty() == false) {
@Override void close(){ if (!isAborted()) { setAborted(execContext.getIoCxt().getIOExceptions()); return; boolean abort = isAborted(); mapOp.close(abort); if (mergeMapOpList.isEmpty() == false) { return; } catch (Exception e) { if (!isAborted()) {
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
rproc = new MapRecordProcessor(jobConf, getContext()); } else { rproc = new ReduceRecordProcessor(jobConf, getContext());
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); super.init(mrReporter, inputs, outputs); checkAbortCondition(); checkAbortCondition(); mergeWorkList.add( (MapWork) cache.retrieve(key, checkAbortCondition(); legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); checkAbortCondition(); createOutputMap(); checkAbortCondition(); checkAbortCondition(); Operator<? extends OperatorDesc> finalOp = getFinalOp(mergeMapOp); if (finalOp instanceof TezDummyStoreOperator) { jconf.set(Utilities.INPUT_NAME, mapWork.getName()); mapOp.initialize(jconf, null); checkAbortCondition(); mapOp.setChildren(jconf); mapOp.passExecContext(execContext);
@Override void close(){ if (!isAborted()) { setAborted(execContext.getIoCxt().getIOExceptions()); return; boolean abort = isAborted(); mapOp.close(abort); if (mergeMapOpList.isEmpty() == false) { return; } catch (Exception e) { if (!isAborted()) {
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
rproc = new MapRecordProcessor(jobConf, getContext()); } else { rproc = new ReduceRecordProcessor(jobConf, getContext());
@Override void run() throws Exception { startAbortChecks(); while (sources[position].pushRecord()) { addRowAndMaybeCheckAbort(); } }
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (MapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } ((TezContext) MapredContext.get()).setRecordSources(sources); }
@Override public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_PROCESSOR); // in case of broadcast-join read the broadcast edge inputs // (possibly asynchronously) LOG.info("Running task: " + getContext().getUniqueIdentifier()); if (isMap) { rproc = new MapRecordProcessor(jobConf, getContext()); } else { rproc = new ReduceRecordProcessor(jobConf, getContext()); } initializeAndRunProcessor(inputs, outputs); }
private Operator<? extends OperatorDesc> getFinalOp(Operator<? extends OperatorDesc> mergeMapOp) { for (Operator<? extends OperatorDesc> childOp : mergeMapOp.getChildOperators()) { if ((childOp.getChildOperators() == null) || (childOp.getChildOperators().isEmpty())) { return childOp; } else { return getFinalOp(childOp); } } return null; }