public static void addNamedOutput(Job job, String name, Class<? extends OutputFormat> outputFormatClass, Class keyClass, Class valueClass) { addNamedOutput(job, name, FormatBundle.forOutput(outputFormatClass), keyClass, valueClass); }
public static void addNamedOutput(Job job, String name, Class<? extends OutputFormat> outputFormatClass, Class keyClass, Class valueClass) { addNamedOutput(job, name, FormatBundle.forOutput(outputFormatClass), keyClass, valueClass); }
public FileTargetImpl(Path path, Class<? extends FileOutputFormat> outputFormatClass, FileNamingScheme fileNamingScheme, Map<String, String> extraConf) { this.path = path; this.formatBundle = FormatBundle.forOutput(outputFormatClass); this.fileNamingScheme = fileNamingScheme; if (extraConf != null && !extraConf.isEmpty()) { for (Map.Entry<String, String> e : extraConf.entrySet()) { formatBundle.set(e.getKey(), e.getValue()); } } }
@Deprecated protected void configureForMapReduce(Job job, Class keyClass, Class valueClass, Class outputFormatClass, Path outputPath, String name) { configureForMapReduce(job, keyClass, valueClass, FormatBundle.forOutput(outputFormatClass), outputPath, name); }
/** * Builds a FormatBundle for DatasetKeyOutputFormat by copying a temp config. * * All properties will be copied from the temporary configuration * * @param conf A Configuration that will be copied * @return a FormatBundle with the contents of conf */ private static FormatBundle<DatasetKeyOutputFormat> outputBundle(Configuration conf) { FormatBundle<DatasetKeyOutputFormat> bundle = FormatBundle .forOutput(DatasetKeyOutputFormat.class); for (Map.Entry<String, String> entry : conf) { bundle.set(entry.getKey(), entry.getValue()); } return bundle; }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType<?>) ptype; FormatBundle bundle = FormatBundle.forOutput(AvroOutputFormat.class); for (Map.Entry<String, String> e : extraConf.entrySet()) { bundle.set(e.getKey(), e.getValue()); } bundle.set("avro.output.schema", atype.getSchema().toString()); AvroMode.fromType(atype).configure(bundle); configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name); }
FormatBundle<TableOutputFormat> bundle = FormatBundle.forOutput(TableOutputFormat.class); bundle.set(TableOutputFormat.OUTPUT_TABLE, table); for (Map.Entry<String, String> e : extraConf.entrySet()) {
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType<?>) ptype; String schemaParam; if (name == null) { schemaParam = PARQUET_AVRO_SCHEMA_PARAMETER; } else { schemaParam = PARQUET_AVRO_SCHEMA_PARAMETER + "." + name; } FormatBundle fb = FormatBundle.forOutput(CrunchAvroParquetOutputFormat.class); for (Map.Entry<String, String> e : extraConf.entrySet()) { fb.set(e.getKey(), e.getValue()); } fb.set(schemaParam, atype.getSchema().toString()); configureForMapReduce(job, Void.class, atype.getTypeClass(), fb, outputPath, name); }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { Converter converter = ptype.getConverter(); Class keyClass = converter.getKeyClass(); Class valueClass = converter.getValueClass(); FormatBundle fb = FormatBundle.forOutput(getOutputFormat(ptype)); for (Map.Entry<String, String> e : extraConf.entrySet()) { fb.set(e.getKey(), e.getValue()); } configureForMapReduce(job, keyClass, valueClass, fb, outputPath, name); }
FormatBundle<TableOutputFormat> bundle = FormatBundle.forOutput(TableOutputFormat.class); bundle.set(TableOutputFormat.OUTPUT_TABLE, table); for (Map.Entry<String, String> e : extraConf.entrySet()) {
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType) ((PTableType) ptype).getValueType(); String schemaParam; if (name == null) { schemaParam = AvroParquetFileTarget.PARQUET_AVRO_SCHEMA_PARAMETER; } else { schemaParam = AvroParquetFileTarget.PARQUET_AVRO_SCHEMA_PARAMETER + "." + name; } FormatBundle fb = FormatBundle.forOutput(AvroParquetPathPerKeyOutputFormat.class); fb.set(schemaParam, atype.getSchema().toString()); configureForMapReduce(job, Void.class, atype.getTypeClass(), fb, outputPath, name); }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { AvroType<?> atype = (AvroType) ((PTableType) ptype).getValueType(); FormatBundle bundle = FormatBundle.forOutput(AvroPathPerKeyOutputFormat.class); bundle.set("avro.output.schema", atype.getSchema().toString()); AvroMode.fromType(atype).configure(bundle); configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle, outputPath, name); }