@Override public boolean commitResource(JobConf conf) throws java.io.IOException { TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this); System.out.println(""); System.out.println(""); System.out.println("RESULTS"); System.out.println("-----------------------"); while (it.hasNext()) { System.out.println(it.next().getTuple()); } System.out.println("-----------------------"); it.close(); return true; } }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }
private TupleEntry timedNext( StepCounters durationCounter, TupleEntryIterator iterator ) { long start = System.currentTimeMillis(); try { return iterator.next(); } finally { flowProcess.increment( durationCounter, System.currentTimeMillis() - start ); } }
public boolean hasNext() { boolean hasNext = grouping.joinIterator.hasNext(); if( !hasNext && !operationCall.isRetainValues() ) tupleEntry.setTuple( valueNulledTuple ); // null out footer entries return hasNext; }
iterator.close();
@Override public void receive( Duct previous, int ordinal, final Grouping<TupleEntry, TupleEntryIterator> grouping ) { try { // we want to null out any 'values' before and after the iterator begins/ends // this allows buffers to emit tuples before next() and when hasNext() return false; final TupleEntry tupleEntry = grouping.joinIterator.getTupleEntry(); incomingEntry = tupleEntry; // if Fields.NONE are declared on the CoGroup, we don't provide arguments, only the joinerClosure if( !tupleEntry.getFields().isNone() ) { final Tuple valueNulledTuple = Tuples.setOnEmpty( tupleEntry, grouping.key ); tupleEntry.setTuple( valueNulledTuple ); operationCall.setArgumentsIterator( createArgumentsIterator( grouping, tupleEntry, valueNulledTuple ) ); } operationCall.setOutputCollector( outputCollector ); operationCall.setJoinerClosure( grouping.joinerClosure ); operationCall.setGroup( grouping.key ); buffer.operate( flowProcess, operationCall ); } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed executing operation: " + every.getOperation(), throwable ), argumentsEntry ); } }
public static List<TupleEntry> getAllTupleEntries(Tap<JobConf, ?, ?> t) throws IOException { TupleEntryIterator iter = t.openForRead(CascadingUtil.get().getFlowProcess()); List<TupleEntry> tuples = new ArrayList<TupleEntry>(); while (iter.hasNext()) { tuples.add(new TupleEntry(iter.next())); } return tuples; }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
public TupleEntry next() { argumentsEntry.setTuple( argumentsBuilder.makeResult( grouping.joinIterator.next().getTuple(), null ) ); return argumentsEntry; }
.hasNext()) { break;
@Override public boolean commitResource(JobConf conf) throws java.io.IOException { TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this); boolean first_time = true; while (it.hasNext()) { TupleEntry tuple = it.next(); results.add(tuple.getTupleCopy()); if (first_time) { fields = tuple.getFields(); first_time = false; } } it.close(); return true; } }
public static List<Tuple> getAllTuples(Tap tap, CascadingUtil util) throws IOException { TupleEntryIterator it = tap.openForRead(util.getFlowProcess()); List<Tuple> ret = new ArrayList<Tuple>(); while (it.hasNext()) { ret.add(new Tuple(it.next().getTuple())); // need to copy it since TapIterator reuses the same tuple object } return ret; }
@Test public void testSinkUnknown() throws IOException { getPlatform().copyFromLocal( inputFileCross ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileCross ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.RESULTS ); Tap sink = getPlatform().getTabDelimitedFile( Fields.UNKNOWN, getOutputPath( "unknownsinks" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 37, null ); TupleEntryIterator iterator = flow.openSink(); String line = iterator.next().getTuple().toString(); assertTrue( "not equal: wrong values: " + line, line.matches( "[0-9]\t[a-z]\t[A-Z]" ) ); iterator.close(); }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
Object result = iterator.next().getObject( 1 );