@Override public RecordReader<AvroKey<GenericRecord>, NullWritable> createRecordReader(InputSplit split, TaskAttemptContext cx) throws IOException { return new CombineFileRecordReader<>((CombineFileSplit) split, cx, AvroKeyCombineFileRecordReader.class); } }
new TaskAttemptContextImpl(conf, taskAttemptID,reporter); CombineFileRecordReader cfrr = new CombineFileRecordReader(combineFileSplit, taskAttemptContext, TextRecordReaderWrapper.class); cfrr.initialize(combineFileSplit,taskAttemptContext); Assert.assertFalse(cfrr.nextKeyValue()); verify(reporter, times(3)).progress(); } finally {
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { if (split instanceof KiteCombineFileSplit) { KiteCombineFileSplit kiteCombineFileSplit = (KiteCombineFileSplit) split; super.initialize(kiteCombineFileSplit.getCombineFileSplit(), context); } else { throw new DatasetOperationException( "Split is not a KiteCombineFileSplit: %s:%s", split.getClass().getCanonicalName(), split); } } }
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
public RecordReader<PairOfStringLong, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader<PairOfStringLong, Text>((CombineFileSplit)split, context, CustomRecordReader.class); }
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
@Override public RecordReader<AvroKey<GenericRecord>, NullWritable> createRecordReader(InputSplit arg0, TaskAttemptContext arg1) throws IOException { return new CombineFileRecordReader<AvroKey<GenericRecord>, NullWritable>((CombineFileSplit) arg0, arg1, AvroKeyCombineFileReader.class); }
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
public RecordReader<LongWritable,Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader<LongWritable,Text>( (CombineFileSplit)split, context, TextRecordReaderWrapper.class); }
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
public RecordReader<WordOffset,Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader<WordOffset, Text>( (CombineFileSplit)split, context, CombineFileLineRecordReader.class); } }
/** * A generic RecordReader that can hand out different recordReaders * for each chunk in the CombineFileSplit. */ public CombineFileRecordReader(CombineFileSplit split, TaskAttemptContext context, Class<? extends RecordReader<K,V>> rrClass) throws IOException { this.split = split; this.context = context; this.idx = 0; this.curReader = null; this.progress = 0; try { rrConstructor = rrClass.getDeclaredConstructor(constructorSignature); rrConstructor.setAccessible(true); } catch (Exception e) { throw new RuntimeException(rrClass.getName() + " does not have valid constructor", e); } initNextRecordReader(); }
@SuppressWarnings({ "rawtypes", "unchecked" }) public RecordReader<K,V> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader((CombineFileSplit)split, context, SequenceFileRecordReaderWrapper.class); }
public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { if (!initNextRecordReader()) { return false; } } return true; }
public RecordReader<LongWritable,Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader<LongWritable,Text>( (CombineFileSplit)split, context, TextRecordReaderWrapper.class); }
/** * A generic RecordReader that can hand out different recordReaders * for each chunk in the CombineFileSplit. */ public CombineFileRecordReader(CombineFileSplit split, TaskAttemptContext context, Class<? extends RecordReader<K,V>> rrClass) throws IOException { this.split = split; this.context = context; this.idx = 0; this.curReader = null; this.progress = 0; try { rrConstructor = rrClass.getDeclaredConstructor(constructorSignature); rrConstructor.setAccessible(true); } catch (Exception e) { throw new RuntimeException(rrClass.getName() + " does not have valid constructor", e); } initNextRecordReader(); }
@Override @SuppressWarnings({ "unchecked", "rawtypes" }) public RecordReader<AvroKey<T>, NullWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader((CombineFileSplit)split, context, DelegatingAvroRecordReader.class); }
/** * A generic RecordReader that can hand out different recordReaders * for each chunk in the CombineFileSplit. */ public CombineFileRecordReader(CombineFileSplit split, TaskAttemptContext context, Class<? extends RecordReader<K,V>> rrClass) throws IOException { this.split = split; this.context = context; this.idx = 0; this.curReader = null; this.progress = 0; try { rrConstructor = rrClass.getDeclaredConstructor(constructorSignature); rrConstructor.setAccessible(true); } catch (Exception e) { throw new RuntimeException(rrClass.getName() + " does not have valid constructor", e); } initNextRecordReader(); }
@Override public RecordReader<CombineFileWritable, OrcStruct> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { return new CombineFileRecordReader<CombineFileWritable, OrcStruct>((CombineFileSplit) split, context, OrcRecordReader.class); }