@Override public void sourceCleanup(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { // in case of a source there's no local client so do all reporting here report(sourceCall.getInput().stats(), flowProcess); report(sourceCall.getInput().repository().stats(), flowProcess); sourceCall.getInput().close(); sourceCall.setContext(null); // used for consistency cleanupClient(flowProcess); }
@Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourcePrepare(flowProcess, sourceCall); Object[] context = new Object[SRC_CTX_SIZE]; context[SRC_CTX_KEY] = sourceCall.getInput().createKey(); context[SRC_CTX_VALUE] = sourceCall.getInput().createValue(); // as the tuple _might_ vary (some objects might be missing), we use a map rather then a collection Settings settings = loadSettings(flowProcess.getConfigCopy(), true); context[SRC_CTX_ALIASES] = CascadingUtils.alias(settings); context[SRC_CTX_OUTPUT_JSON] = settings.getOutputAsJson(); sourceCall.setContext(context); }
Object[] context = sourceCall.getContext(); if (!sourceCall.getInput().next(context[SRC_CTX_KEY], context[1])) { return false;
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { ScrollQuery query = sourceCall.getInput();
private boolean sourceReadInput( SourceCall<Object[], RecordReader> sourceCall ) throws IOException { Object[] context = sourceCall.getContext(); return sourceCall.getInput().next( context[ 0 ], context[ 1 ] ); }
@Override public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); }
@Override public void sourcePrepare( FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader<TupleWrapper, NullWritable>> sourceCall ) { sourceCall.setContext( new Object[ 2 ] ); sourceCall.getContext()[ 0 ] = sourceCall.getInput().createKey(); sourceCall.getContext()[ 1 ] = sourceCall.getInput().createValue(); }
/** * This sets up the state between succesive calls to source */ @Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) { //Hadoop sets a key value pair: sourceCall.setContext(new Object[2]); sourceCall.getContext()[0] = sourceCall.getInput().createKey(); sourceCall.getContext()[1] = sourceCall.getInput().createValue(); } }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<JobConf> fp, SourceCall<Object[], RecordReader> sc) throws IOException { Container<Tuple> value = (Container<Tuple>) sc.getInput().createValue(); boolean hasNext = sc.getInput().next(null, value); if (!hasNext) { return false; } // Skip nulls if (value == null) { return true; } sc.getIncomingEntry().setTuple(value.get()); return true; }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<? extends JobConf> fp, SourceCall<Object[], RecordReader> sc) throws IOException { Container<T> value = (Container<T>) sc.getInput().createValue(); boolean hasNext = sc.getInput().next(null, value); if (!hasNext) { return false; } // Skip nulls if (value == null) { return true; } sc.getIncomingEntry().setTuple(new Tuple(value.get())); return true; }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( sourceCall.getContext() == null ) sourceCall.setContext( new Object[ 3 ] ); sourceCall.getContext()[ 0 ] = sourceCall.getInput().createKey(); sourceCall.getContext()[ 1 ] = sourceCall.getInput().createValue(); sourceCall.getContext()[ 2 ] = Charset.forName( charsetName ); }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( sourceCall.getContext() == null ) sourceCall.setContext( new Object[ 3 ] ); sourceCall.getContext()[ 0 ] = sourceCall.getInput().createKey(); sourceCall.getContext()[ 1 ] = sourceCall.getInput().createValue(); sourceCall.getContext()[ 2 ] = Charset.forName( charsetName ); }
@Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourcePrepare(flowProcess, sourceCall); Object[] context = new Object[SRC_CTX_SIZE]; context[SRC_CTX_KEY] = sourceCall.getInput().createKey(); context[SRC_CTX_VALUE] = sourceCall.getInput().createValue(); // as the tuple _might_ vary (some objects might be missing), we use a map rather then a collection Settings settings = loadSettings(flowProcess.getConfigCopy(), true); context[SRC_CTX_ALIASES] = CascadingUtils.alias(settings); context[SRC_CTX_OUTPUT_JSON] = settings.getOutputAsJson(); sourceCall.setContext(context); }
@Override public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader<TupleWrapper, NullWritable>> sourceCall) throws IOException { TupleWrapper key = (TupleWrapper) sourceCall.getContext()[ 0 ]; NullWritable value = (NullWritable) sourceCall.getContext()[ 1 ]; boolean result = sourceCall.getInput().next( key, value ); if( !result ) return false; sourceCall.getIncomingEntry().setTuple(key.tuple); return true; }
@Override public boolean source(FlowProcess<JobConf> prcs, SourceCall<Object[], RecordReader> sourceCall) throws IOException { Object key = sourceCall.getContext()[0]; Object value = sourceCall.getContext()[1]; boolean result = sourceCall.getInput().next(key, value); if (!result) { return false; } sourceCall.getIncomingEntry().setTuple(new Tuple(key, value)); return true; }
@Override public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException { boolean success = sourceCall.getInput().next( sourceCall.getContext()[ 0 ], sourceCall.getContext()[ 1 ] ); if( !success ) return false; if( sourceCall.getContext()[ 1 ].toString().matches( "^\\s*#.*$" ) ) return source( flowProcess, sourceCall ); sourceHandleInput( sourceCall ); return true; } }