public synchronized boolean nextKeyValue() throws IOException, InterruptedException { int count = 0; data.clear(); while (sequenceFileRecordReader.nextKeyValue()) { JSONObject obj = new JSONObject(sequenceFileRecordReader.getCurrentValue().toString()); data.add(obj); count++; if (count == batchSize) { break; } } return (!data.isEmpty()); }
@Override public Tuple getNext() throws IOException { boolean next; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) return null; Object value = reader.getCurrentValue(); if (value == null) { return null; } if(!(value instanceof Tuple)) { return null; } return (Tuple) value; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true; if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true;
if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true; if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true;
if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true; if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true;
if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true; if (chunk.getReader().nextKeyValue()) { ++numRecordsProcessedByThisMap; return true;
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
@Override public Tuple getNext() throws IOException { boolean next = false; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) { return null; } key = reader.getCurrentKey(); value = reader.getCurrentValue(); if (keyType == DataType.UNKNOWN && key != null) { setKeyType(key.getClass()); } if (valType == DataType.UNKNOWN && value != null) { setValueType(value.getClass()); } mProtoTuple.add(translateWritableToPigDataType(key, keyType)); mProtoTuple.add(translateWritableToPigDataType(value, valType)); Tuple t = mTupleFactory.newTuple(mProtoTuple); mProtoTuple.clear(); return t; }
private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) { try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { Path path = new Path(fileName); TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(), new JobID("readJob", 0), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); List<KV<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add( KV.of( new Text(reader.getCurrentKey().toString()), new LongWritable(reader.getCurrentValue().get()))); } return result.stream(); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public Tuple getNext() throws IOException { boolean next; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) return null; Object value = reader.getCurrentValue(); if (value == null) { return null; } if (!(value instanceof StockPriceWritable)) { return null; } StockPriceWritable w = (StockPriceWritable) value; return TupleFactory.getInstance().newTuple(Arrays.asList( w.getSymbol(), w.getDate(), w.getOpen(), w.getHigh(), w.getLow(), w.getClose(), w.getVolume(), w.getAdjClose() )); }
@Override public Function<String, Stream<Pair<Text, LongWritable>>> extractOutputFunction( String outputDir, Configuration conf) { return part -> ExceptionUtils.unchecked( () -> { try (final SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { final Path path = new Path(outputDir + "/" + part); final TaskAttemptContext taskContext = HadoopUtils.createTaskContext(new Configuration(), HadoopUtils.getJobID(), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); final List<Pair<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add(Pair.of(reader.getCurrentKey(), reader.getCurrentValue())); } return result.stream(); } }); }