@Override public Target outputConf(String key, String value) { formatBundle.set(key, value); return this; }
@Override public Target outputConf(String key, String value) { formatBundle.set(key, value); return this; }
@Override public Source<E> inputConf(String key, String value) { formatBundle.set(key, value); return this; }
@Override public Source<HCatRecord> inputConf(String key, String value) { bundle.set(key, value); return this; }
@Override public Source<T> inputConf(String key, String value) { inputBundle.set(key, value); return this; }
@Override public Source<Pair<BytesWritable, BytesWritable>> inputConf(String key, String value) { inputBundle.set(key, value); return this; }
@Override public Source<Pair<ImmutableBytesWritable, Result>> inputConf(String key, String value) { inputBundle.set(key, value); return this; }
/** * Populates the {@code bundle} with mode specific settings for the specific {@link FormatBundle}. * @param bundle the bundle to populate. */ public void configure(FormatBundle bundle) { bundle.set(AVRO_MODE_PROPERTY, this.modeType.toString()); if (factory != null) { bundle.set(propName, factory.getClass().getName()); } }
private static <S> FormatBundle getBundle(AvroType<S> ptype, AvroMode mode) { FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class) .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect())) .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString()) .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName()) .set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString()); mode.configure(bundle); return bundle; }
private static FormatBundle getBundle(int linesPerTask) { FormatBundle bundle = FormatBundle.forInput(NLineInputFormat.class); bundle.set(NLineInputFormat.LINES_PER_MAP, String.valueOf(linesPerTask)); return bundle; }
private static <S> FormatBundle getBundle(AvroType<S> ptype) { FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class) .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect())) .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString()) .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName()); return bundle; }
HFileSource(List<Path> paths, Scan scan) { super(paths, KEY_VALUE_PTYPE, createInputFormatBundle(scan) // "combine file" is not supported by HFileInputFormat, as it overrides listStatus(). .set(RuntimeParameters.DISABLE_COMBINE_FILE, "true")); }
private static <S> FormatBundle<OrcCrunchInputFormat> getBundle(int[] readColumns) { FormatBundle<OrcCrunchInputFormat> fb = FormatBundle.forInput(OrcCrunchInputFormat.class); if (readColumns != null) { // setting configurations for column pruning fb.set(HIVE_READ_ALL_COLUMNS, "false"); fb.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, getColumnIdsStr(readColumns)); } return fb; }
public TextFileSource(List<Path> paths, PType<T> ptype) { super(paths, ptype, getInputFormat(ptype)); inputBundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString()); }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType<?>) ptype; FormatBundle bundle = FormatBundle.forOutput(AvroOutputFormat.class); for (Map.Entry<String, String> e : extraConf.entrySet()) { bundle.set(e.getKey(), e.getValue()); } bundle.set("avro.output.schema", atype.getSchema().toString()); AvroMode.fromType(atype).configure(bundle); configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name); }
private static FormatBundle<HFileInputFormat> createInputFormatBundle(Scan scan) { FormatBundle<HFileInputFormat> bundle = FormatBundle.forInput(HFileInputFormat.class); if (!Objects.equal(scan.getStartRow(), HConstants.EMPTY_START_ROW)) { bundle.set(HFileInputFormat.START_ROW_KEY, Hex.encodeHexString(scan.getStartRow())); } if (!Objects.equal(scan.getStopRow(), HConstants.EMPTY_END_ROW)) { bundle.set(HFileInputFormat.STOP_ROW_KEY, Hex.encodeHexString(scan.getStopRow())); } return bundle; }
private static FormatBundle<HFileInputFormat> createInputFormatBundle(Scan scan) { FormatBundle<HFileInputFormat> bundle = FormatBundle.forInput(HFileInputFormat.class); if (!Objects.equal(scan.getStartRow(), HConstants.EMPTY_START_ROW)) { bundle.set(HFileInputFormat.START_ROW_KEY, Hex.encodeHexString(scan.getStartRow())); } if (!Objects.equal(scan.getStopRow(), HConstants.EMPTY_END_ROW)) { bundle.set(HFileInputFormat.STOP_ROW_KEY, Hex.encodeHexString(scan.getStopRow())); } return bundle; }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType) ((PTableType) ptype).getValueType(); FormatBundle bundle = FormatBundle.forOutput(AvroPathPerKeyOutputFormat.class); bundle.set("avro.output.schema", atype.getSchema().toString()); AvroMode.fromType(atype).configure(bundle); configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name); }