/** {@inheritDoc} */ @Override protected Schema initSchema(TaskAttemptContext context) { boolean isMapOnly = context.getNumReduceTasks() == 0; return isMapOnly ? AvroJob.getMapOutputKeySchema(context .getConfiguration()) : AvroJob.getOutputKeySchema(context .getConfiguration()); } }
boolean isMapOnly = context.getNumReduceTasks() == 0; if (isMapOnly) { Schema mapOutputSchema = AvroJob.getMapOutputKeySchema(conf);
.andReturn(TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0")) .anyTimes(); expect(context.getNumReduceTasks()).andReturn(1);
@Override public int getNumReduceTasks() { return delegate.getNumReduceTasks(); }
if (redcount < 0) throw new IOException("Invalid reduce count: " + redcount); final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
taskContext.getNumReduceTasks() : conf.getInt(IREDUCE_TASKS_COUNT, 1); int sleepCount = totalIReduces == 0?
public Datafile getDatafile(TaskAttemptContext context, String folder) throws IOException { int task = ContextTools.getTaskID(context); Configuration conf = context.getConfiguration(); FileSystem fs = HDFSPath.getFS(conf); String filename = conf.get(SINGLEFILE); if (filename == null || context.getNumReduceTasks() > 1) { HDFSPath dir = new HDFSPath(conf, folder); filename = dir.getFilename(PrintTools.sprintf("%s.%05d", "part", task)); } return new Datafile(fs, filename); }
if (redcount < 0) throw new IOException("Invalid reduce count: " + redcount); final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
@Override public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext job) throws IOException,InterruptedException { Configuration conf = job.getConfiguration(); boolean isMapOnly = job.getNumReduceTasks() == 0; Schema schema = isMapOnly ? AvroJob.getMapOutputSchema(conf) : AvroJob.getOutputSchema(conf); final DataFileWriter<T> writer = new DataFileWriter<T>(new ReflectDatumWriter<T>()); configureDataFileWriter(writer, job); Path path = getDefaultWorkFile(job,EXT); writer.create(schema, path.getFileSystem(job.getConfiguration()).create(path)); return new RecordWriter<AvroWrapper<T>, NullWritable>() { @Override public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { writer.append(wrapper.datum()); } @Override public void close(TaskAttemptContext context) throws IOException { writer.close(); } }; }
@Override public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext job) throws IOException,InterruptedException { Configuration conf = job.getConfiguration(); boolean isMapOnly = job.getNumReduceTasks() == 0; Schema schema = isMapOnly ? AvroJob.getMapOutputSchema(conf) : AvroJob.getOutputSchema(conf); final DataFileWriter<T> writer = new DataFileWriter<T>(new ReflectDatumWriter<T>()); configureDataFileWriter(writer, job); Path path = getDefaultWorkFile(job,EXT); writer.create(schema, path.getFileSystem(job.getConfiguration()).create(path)); return new RecordWriter<AvroWrapper<T>, NullWritable>() { @Override public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { writer.append(wrapper.datum()); } @Override public void close(TaskAttemptContext context) throws IOException { writer.close(); } }; }
/** {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); // Get the writer schema. Schema writerSchema = AvroJob.getOutputKeySchema(conf); boolean isMapOnly = context.getNumReduceTasks() == 0; if (isMapOnly) { Schema mapOutputSchema = AvroJob.getMapOutputKeySchema(conf); if (mapOutputSchema != null) { writerSchema = mapOutputSchema; } } if (null == writerSchema) { throw new IOException( "AvroKeyOutputFormat requires an output schema. Use AvroJob.setOutputKeySchema()."); } GenericData dataModel = AvroSerialization.createDataModel(conf); return mRecordWriterFactory.create (writerSchema, dataModel, getCompressionCodec(context), getAvroFileOutputStream(context), getSyncInterval(context)); } }