input = queryBuilder.build(client, new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), mapping, settings))); return new TupleEntrySchemeIterator<Properties, ScrollQuery>(flowProcess, getScheme(), input, getIdentifier());
@Override public boolean hasNext() { return childIterator.hasNext(); }
@Override public void remove() { childIterator.remove(); } }
@Test public void testHasNextWithException() throws Exception { FlowProcess<?> flowProcess = FlowProcess.NULL; Scheme<?, ?, ?, ?, ?> scheme = new MockedScheme(); CloseableIterator<Object> inputIterator = new MockedSingleValueCloseableIterator( new Object() ); TupleEntrySchemeIterator<?, ?> iterator = new TupleEntrySchemeIterator( flowProcess, null, scheme, inputIterator ); assertTrue( iterator.hasNext() ); assertTrue( iterator.hasNext() ); }
sourceCall = createSourceCall(); sourceCall.setIncomingEntry( getTupleEntry() ); sourceCall.setInput( wrapInput( inputIterator.next() ) );
@Override public Tuple next() { String currentFile = getCurrentFile(); if( this.currentFile == null || !this.currentFile.equals( currentFile ) ) { this.currentFile = currentFile; try { String childIdentifier = new Path( currentFile ).getParent().toString(); // drop part-xxxx partition.toTuple( childIdentifier.substring( parentIdentifier.length() + 1 ), partitionEntry ); } catch( Exception exception ) { throw new TapException( "unable to parse partition given parent: " + parentIdentifier + " and child: " + currentFile, exception ); } base = TupleViews.createOverride( sourceFields, partitionEntry.getFields() ); TupleViews.reset( base, Tuple.size( sourceFields.size() ), partitionEntry.getTuple() ); view = TupleViews.createOverride( sourceFields, childIterator.getFields() ); } Tuple tuple = childIterator.next().getTuple(); TupleViews.reset( view, base, tuple ); return view; }
private TupleEntry getNext() throws IOException { Tuples.asModifiable( sourceCall.getIncomingEntry().getTuple() ); hasWaiting = scheme.source( flowProcess, sourceCall ); while( !hasWaiting && inputIterator.hasNext() ) { sourceCall.setInput( wrapInput( inputIterator.next() ) ); try { scheme.sourceRePrepare( flowProcess, sourceCall ); } catch( IOException exception ) { throw new TupleException( "unable to prepare source for input identifier: " + loggableIdentifier.get(), exception ); } Tuples.asModifiable( sourceCall.getIncomingEntry().getTuple() ); hasWaiting = scheme.source( flowProcess, sourceCall ); } return getTupleEntry(); }
@Override protected RecordReader wrapInput( RecordReader recordReader ) { if( measuredRecordReader == null ) measuredRecordReader = new MeasuredRecordReader( getFlowProcess(), SliceCounters.Read_Duration ); measuredRecordReader.setRecordReader( super.wrapInput( recordReader ) ); return measuredRecordReader; } }
@Override public Tuple next() { Tuple tuple = childIterator.next().getTuple(); TupleViews.reset( view, base, tuple ); return view; }
public PartitionTupleEntryIterator( Fields sourceFields, Partition partition, String parentIdentifier, String childIdentifier, TupleEntrySchemeIterator schemeIterator ) { this.childIterator = schemeIterator; TupleEntry partitionEntry = new TupleEntry( partition.getPartitionFields(), Tuple.size( partition.getPartitionFields().size() ) ); try { partition.toTuple( childIdentifier.substring( parentIdentifier.length() + 1 ), partitionEntry ); } catch( Exception exception ) { throw new TapException( "unable to parse partition given parent: " + parentIdentifier + " and child: " + childIdentifier ); } base = TupleViews.createOverride( sourceFields, partitionEntry.getFields() ); TupleViews.reset( base, Tuple.size( sourceFields.size() ), partitionEntry.getTuple() ); view = TupleViews.createOverride( sourceFields, childIterator.getFields() ); }
@Override public Tuple next() { String currentFile = getCurrentFile(); if( this.currentFile == null || !this.currentFile.equals( currentFile ) ) { this.currentFile = currentFile; try { String childIdentifier = new Path( currentFile ).getParent().toString(); // drop part-xxxx partition.toTuple( childIdentifier.substring( parentIdentifier.length() + 1 ), partitionEntry ); } catch( Exception exception ) { throw new TapException( "unable to parse partition given parent: " + parentIdentifier + " and child: " + currentFile, exception ); } base = TupleViews.createOverride( sourceFields, partitionEntry.getFields() ); TupleViews.reset( base, Tuple.size( sourceFields.size() ), partitionEntry.getTuple() ); view = TupleViews.createOverride( sourceFields, childIterator.getFields() ); } Tuple tuple = childIterator.next().getTuple(); TupleViews.reset( view, base, tuple ); return view; }
@Override protected RecordReader wrapInput( RecordReader recordReader ) { if( measuredRecordReader == null ) measuredRecordReader = new MeasuredRecordReader( getFlowProcess(), SliceCounters.Read_Duration ); measuredRecordReader.setRecordReader( super.wrapInput( recordReader ) ); return measuredRecordReader; } }
@Override public TupleEntryIterator openForRead( FlowProcess<? extends Properties> flowProcess, InputStream inputStream ) throws IOException { return new TupleEntrySchemeIterator<Properties, InputStream>( flowProcess, this, getScheme(), System.in ); }
@Override public boolean hasNext() { return childIterator.hasNext(); }
@Override public void remove() { childIterator.remove(); } }
@Override public TupleEntryIterator openForRead(FlowProcess<Properties> flowProcess, Resource input) throws IOException { InputStream in = (input != null ? input.getInputStream() : resource.getInputStream()); return new TupleEntrySchemeIterator<Properties, InputStream>(flowProcess, getScheme(), in, getIdentifier()); }
@Override public boolean hasNext() { return childIterator.hasNext(); }
@Override public void remove() { childIterator.remove(); } }
@Override public TupleEntryIterator openForRead(FlowProcess<Properties> flowProcess, InputStream input) throws IOException { InputStream in = (input == null ? source : input); return new TupleEntrySchemeIterator<Properties, InputStream>(flowProcess, getScheme(), in, getIdentifier()); }
@Override protected TupleEntrySchemeIterator createTupleEntrySchemeIterator( FlowProcess<? extends Properties> flowProcess, Tap parent, String path, InputStream input ) throws IOException { if( parent instanceof TapWith ) return (TupleEntrySchemeIterator) ( (TapWith) parent ) .withChildIdentifier( path ) .asTap().openForRead( flowProcess, input ); if( input == null ) input = new FileInputStream( path ); return new TupleEntrySchemeIterator( flowProcess, parent, parent.getScheme(), input, path ); } }