/** * Implementation of RecordReader::getCurrentKey(). * @return The key of the current record. (i.e. the source-path.) * @throws IOException * @throws InterruptedException */ @Override public K getCurrentKey() throws IOException, InterruptedException { return chunk.getReader().getCurrentKey(); }
/** * Implementation of RecordReader::getCurrentKey(). * * @return The key of the current record. (i.e. the source-path.) * @throws IOException, on failure. * @throws InterruptedException */ @Override public K getCurrentKey() throws IOException, InterruptedException { return chunk.getReader().getCurrentKey(); }
/** * Implementation of RecordReader::getCurrentKey(). * @return The key of the current record. (i.e. the source-path.) * @throws IOException * @throws InterruptedException */ @Override public K getCurrentKey() throws IOException, InterruptedException { return chunk.getReader().getCurrentKey(); }
/** * Implementation of RecordReader::getCurrentKey(). * * @return The key of the current record. (i.e. the source-path.) * @throws IOException, on failure. * @throws InterruptedException */ @Override public K getCurrentKey() throws IOException, InterruptedException { return chunk.getReader().getCurrentKey(); }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
@Override public Tuple getNext() throws IOException { boolean next = false; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) { return null; } key = reader.getCurrentKey(); value = reader.getCurrentValue(); if (keyType == DataType.UNKNOWN && key != null) { setKeyType(key.getClass()); } if (valType == DataType.UNKNOWN && value != null) { setValueType(value.getClass()); } mProtoTuple.add(translateWritableToPigDataType(key, keyType)); mProtoTuple.add(translateWritableToPigDataType(value, valType)); Tuple t = mTupleFactory.newTuple(mProtoTuple); mProtoTuple.clear(); return t; }
private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) { try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { Path path = new Path(fileName); TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(), new JobID("readJob", 0), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); List<KV<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add( KV.of( new Text(reader.getCurrentKey().toString()), new LongWritable(reader.getCurrentValue().get()))); } return result.stream(); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public Function<String, Stream<Pair<Text, LongWritable>>> extractOutputFunction( String outputDir, Configuration conf) { return part -> ExceptionUtils.unchecked( () -> { try (final SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { final Path path = new Path(outputDir + "/" + part); final TaskAttemptContext taskContext = HadoopUtils.createTaskContext(new Configuration(), HadoopUtils.getJobID(), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); final List<Pair<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add(Pair.of(reader.getCurrentKey(), reader.getCurrentValue())); } return result.stream(); } }); }