Configuration updatedConf = mrInput.getConfigUpdates(); if (updatedConf != null) { for (Map.Entry<String, String> entry : updatedConf) {
if (li.size() == 1) { theMRInput = (MRInputLegacy) li.get(0); theMRInput.init(); } else { throw new IllegalArgumentException("MRInputs count is expected to be 1");
dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) .setCustomInitializerDescriptor(descriptor).build(); } else { } else { dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build();
mrInput.init(); UnorderedKVOutput kvOutput = (UnorderedKVOutput) lo; Configuration updatedConf = mrInput.getConfigUpdates(); Text srcFile = new Text(); srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR"); KeyValueReader kvReader = mrInput.getReader(); KeyValueWriter kvWriter = kvOutput.getWriter();
input.init(); Configuration incrementalConf = input.getConfigUpdates(); if (incrementalConf != null) { for (Entry<String, String> entry : incrementalConf) {
@Override void run() throws Exception { KeyValueReader reader = mrInput.getReader(); //process records until done while (reader.next()) { boolean needMore = processRow(reader.getCurrentKey(), reader.getCurrentValue()); if (!needMore || isAborted()) { break; } } }
if (initEvent == null) { if (LOG.isDebugEnabled()) { LOG.debug(getContext().getSourceVertexName() + " awaiting init event before initializing record reader"); initFromEvent(initEvent); inited = true; } else {
@Private protected void initializeInternal() throws IOException { LOG.info(getContext().getSourceVertexName() + " MRInputLegacy deferring initialization"); }
@LimitedPrivate("hive") public void init() throws IOException { super.initializeInternal(); checkAndAwaitRecordReaderInitialization(); }
@Override void run() throws Exception { KeyValueReader reader = mrInput.getReader(); //process records until done while (reader.next()) { boolean needMore = processRow(reader.getCurrentKey(), reader.getCurrentValue()); if (!needMore || isAborted()) { break; } } }
theMRInput.init(); } else { String alias = mapWork.getAliasToWork().keySet().iterator().next();
Configuration updatedConf = mrInput.getConfigUpdates(); if (updatedConf != null) { for (Map.Entry<String, String> entry : updatedConf) {
dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) .setCustomInitializerDescriptor(descriptor).build(); } else { } else { dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build();
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
theMRInput.init(); } else { String alias = mapWork.getAliasToWork().keySet().iterator().next();
legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); if (updatedConf != null) { for (Entry<String, String> entry : updatedConf) {
.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir, true); } else { dataSource = MRInputLegacy.createConfigBuilder(mapStageConf, SleepInputFormat.class) .generateSplitsInAM(generateSplitsInAM).build();
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
if (li.size() == 1) { theMRInput = (MRInputLegacy) li.get(0); theMRInput.init(); } else { throw new IllegalArgumentException("MRInputs count is expected to be 1");
legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); if (updatedConf != null) { for (Entry<String, String> entry : updatedConf) {