@Override void run() throws Exception { KeyValueReader reader = mrInput.getReader(); //process records until done while (reader.next()) { boolean needMore = processRow(reader.getCurrentKey(), reader.getCurrentValue()); if (!needMore || isAborted()) { break; } } }
if (li.size() == 1) { theMRInput = (MRInputLegacy) li.get(0); theMRInput.init(); } else { throw new IllegalArgumentException("MRInputs count is expected to be 1");
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) .setCustomInitializerDescriptor(descriptor).build(); } else { MultiMRInput.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build(); } else { dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build();
legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); if (updatedConf != null) { for (Entry<String, String> entry : updatedConf) { skipRead = true; } else { Collection<KeyValueReader> keyValueReaders = multiMRInput.getKeyValueReaders(); if ((keyValueReaders == null) || (keyValueReaders.isEmpty())) { l4j.info("Key value readers are null or empty and hence skipping read. "
LOG.info("Going through each work and adding MultiMRInput"); mergeVx.addDataSource(mapWork.getName(), MultiMRInput.createConfigBuilder(conf, HiveInputFormat.class).build());
/** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * @param conf Configuration for the {@link MRInputLegacy} * @param inputFormat InputFormat derived class * @return {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} */ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class<?> inputFormat) { return MRInput.createConfigBuilder(conf, inputFormat).setInputClassName(MRInputLegacy.class.getName()); }
Configuration updatedConf = mrInput.getConfigUpdates(); if (updatedConf != null) { for (Map.Entry<String, String> entry : updatedConf) {
private NewRecordReader(MRInput in) throws IOException { this.in = in; this.reader = in.getReader(); }
@LimitedPrivate("hive") public void init() throws IOException { super.initializeInternal(); checkAndAwaitRecordReaderInitialization(); }
dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) .setCustomInitializerDescriptor(descriptor).build(); } else { MultiMRInput.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build(); } else { dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build();
legacyMRInput = getMRInput(inputs); if (legacyMRInput != null) { Configuration updatedConf = legacyMRInput.getConfigUpdates(); if (updatedConf != null) { for (Entry<String, String> entry : updatedConf) { skipRead = true; } else { Collection<KeyValueReader> keyValueReaders = multiMRInput.getKeyValueReaders(); if ((keyValueReaders == null) || (keyValueReaders.isEmpty())) { l4j.info("Key value readers are null or empty and hence skipping read. "
if (li.size() == 1) { theMRInput = (MRInputLegacy) li.get(0); theMRInput.init(); } else { throw new IllegalArgumentException("MRInputs count is expected to be 1");
private void initializeMapRecordSources() throws Exception { int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); KeyValueReader reader = null; if (mainWorkMultiMRInput != null) { reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); } else { reader = legacyMRInput.getReader(); } sources[position].init(jconf, mapOp, reader); for (AbstractMapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); sources[tag] = new MapRecordSource(); String inputName = mapOp.getConf().getName(); MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection<KeyValueReader> kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); if (kvReaders.size() > 0) { reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } } ((TezContext) MapredContext.get()).setRecordSources(sources); }
LOG.info("Going through each work and adding MultiMRInput"); mergeVx.addDataSource(mapWork.getName(), MultiMRInput.createConfigBuilder(conf, HiveInputFormat.class).build());
/** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} for a FileInputFormat * @param conf Configuration for the {@link MRInputLegacy} * @param inputFormat FileInputFormat derived class * @param inputPaths Comma separated input paths * @return {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} */ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class<?> inputFormat, String inputPaths) { return MRInput.createConfigBuilder(conf, inputFormat, inputPaths).setInputClassName( MRInputLegacy.class.getName()); }
Configuration updatedConf = mrInput.getConfigUpdates(); if (updatedConf != null) { for (Map.Entry<String, String> entry : updatedConf) {
@Override void run() throws Exception { KeyValueReader reader = mrInput.getReader(); //process records until done while (reader.next()) { boolean needMore = processRow(reader.getCurrentKey(), reader.getCurrentValue()); if (!needMore || isAborted()) { break; } } }
theMRInput.init(); } else { String alias = mapWork.getAliasToWork().keySet().iterator().next();
theMRInput.init(); } else { String alias = mapWork.getAliasToWork().keySet().iterator().next();