/** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected Schema initSchema(TaskAttemptContext context) { AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory( context.getConfiguration()); keyConverter = converterFactory.create((Class<K>) context .getOutputKeyClass()); valueConverter = converterFactory.create((Class<V>) context .getOutputValueClass()); // Create the generic record schema for the key/value pair. return AvroKeyValue.getSchema( keyConverter.getWriterSchema(), valueConverter.getWriterSchema()); }
/** {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(conf); AvroDatumConverter<K, ?> keyConverter = converterFactory.create( (Class<K>) context.getOutputKeyClass()); AvroDatumConverter<V, ?> valueConverter = converterFactory.create( (Class<V>) context.getOutputValueClass()); GenericData dataModel = AvroSerialization.createDataModel(conf); OutputStream out = getAvroFileOutputStream(context); try { return new AvroKeyValueRecordWriter<>(keyConverter, valueConverter, dataModel, getCompressionCodec(context), out, getSyncInterval(context)); } catch (IOException e) { out.close(); throw e; } } }
.withConfiguration(conf) .withOutputPath(outputFile) .withKeyClass(context.getOutputKeyClass()) .withValueClass(context.getOutputValueClass()) .withProgressable(context)
@Override public Class<?> getOutputKeyClass() { return delegate.getOutputKeyClass(); }
private SequenceFile.Writer getStore(String fieldValue) throws IOException { SequenceFile.Writer writer = storeMap.get(fieldValue); if (writer == null) { writer = createSequenceWriter(fieldValue,ctx.getOutputKeyClass(),ctx.getOutputValueClass()); storeMap.put(fieldValue, writer); } return writer; }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context ) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context ) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Override public void collect(K key, V value) throws IOException { try { if (recordWriter == null) { if(taskAttemptContext.getOutputKeyClass() == null) { when(taskAttemptContext.getOutputKeyClass()).thenReturn((Class)key.getClass()); } if(taskAttemptContext.getOutputValueClass() == null) { when(taskAttemptContext.getOutputValueClass()).thenReturn((Class)value.getClass()); } if(taskAttemptContext.getTaskAttemptID() == null) { when(taskAttemptContext.getTaskAttemptID()).thenReturn(TASK_ID); } recordWriter = outputFormat.getRecordWriter(taskAttemptContext); } recordWriter.write(key, value); } catch (InterruptedException e) { throw new IllegalStateException(e); } }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context ) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context ) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context ) throws IOException, InterruptedException { final SequenceFile.Writer out = getSequenceWriter(context, context.getOutputKeyClass(), context.getOutputValueClass()); return new RecordWriter<K, V>() { public void write(K key, V value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
context.getOutputKeyClass().asSubclass(WritableComparable.class), context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);
context.getOutputKeyClass().asSubclass(WritableComparable.class), context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);
context.getOutputKeyClass().asSubclass(WritableComparable.class), context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter( TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; if (getCompressOutput(context)) { // find the kind of compression to do compressionType = SequenceFileOutputFormat.getOutputCompressionType(context); // find the right codec Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); } Path file = getDefaultWorkFile(context, ""); FileSystem fs = file.getFileSystem(conf); // ignore the progress parameter, since MapFile is local final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(), context.getOutputKeyClass().asSubclass(WritableComparable.class), context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context); return new RecordWriter<WritableComparable<?>, Writable>() { public void write(WritableComparable<?> key, Writable value) throws IOException { out.append(key, value); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
context.getOutputKeyClass().asSubclass(WritableComparable.class), context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);
/** {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException { AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory( context.getConfiguration()); AvroDatumConverter<K, ?> keyConverter = converterFactory.create( (Class<K>) context.getOutputKeyClass()); AvroDatumConverter<V, ?> valueConverter = converterFactory.create( (Class<V>) context.getOutputValueClass()); return new AvroKeyValueWithMetadataRecordWriter<K, V>(keyConverter, valueConverter, getCompressionCodec(context), getAvroFileOutputStream(context), context.getConfiguration()); } }
context.getOutputKeyClass(), context.getOutputValueClass(), compressionType,
/** {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(conf); AvroDatumConverter<K, ?> keyConverter = converterFactory.create( (Class<K>) context.getOutputKeyClass()); AvroDatumConverter<V, ?> valueConverter = converterFactory.create( (Class<V>) context.getOutputValueClass()); GenericData dataModel = AvroSerialization.createDataModel(conf); return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter, dataModel, getCompressionCodec(context), getAvroFileOutputStream(context), getSyncInterval(context)); } }