@Override public TupleEntryCollector openForWrite(FlowProcess<Object> flowProcess, Object output) throws IOException { initInnerTapIfNotSetFromFlowProcess(flowProcess); return actualTap.openForWrite(flowProcess, output); }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { return trap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Config> flowProcess, Output output ) throws IOException { return original.openForWrite( flowProcess, output ); }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends Config> flowProcess ) throws IOException { return original.openForWrite( flowProcess ); }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { return trap.openForWrite( this ); }
@Override public TupleEntryCollector openTapForWrite(Tap tap) throws IOException { return tap.openForWrite( this, null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends TConfig> flowProcess ) throws IOException { return original.openForWrite( processProvider.apply( flowProcess ) ); }
@Override public TupleEntryCollector openForWrite( FlowProcess<? extends TConfig> flowProcess, TOutput output ) throws IOException { return original.openForWrite( processProvider.apply( flowProcess ), null ); }
@Override public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException { return tap.openForWrite( getFlowProcess() ); }
@Override public TupleEntryCollector openForWrite(FlowProcess<Object> flowProcess, Object output) throws IOException { initInnerTapIfNotSetFromFlowProcess(flowProcess); return actualTap.openForWrite(flowProcess, output); }
private void writeToHadoopTap(Tap<?, ?, ?> tap) throws IOException { @SuppressWarnings("unchecked") Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) tap; JobConf conf = new JobConf(); HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf); hadoopTap.sinkConfInit(flowProcess, conf); TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess); for (TupleEntry tuple : data.asTupleEntryList()) { collector.add(tuple); } collector.close(); }
public static void writeToStore(CascadingUtil util, TupleDataStore store, Tuple ... tuples) throws IOException{ TupleEntryCollector output = store.getTap().openForWrite(util.getFlowProcess()); for(Tuple t: tuples){ output.add(t); } output.close(); }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
private void writeValuesTuple( PlatformBroker platformBroker, Ref head ) throws IOException { SchemaCatalogManager catalog = platformBroker.getCatalogManager(); String identifier = getIdentifierFor( platformBroker, head ); TableDef tableDef = createTableFor( catalog, head, identifier ); TupleEntryCollector collector = catalog.createTapFor( tableDef, SinkMode.KEEP ).openForWrite( platformBroker.getFlowProcess() ); for( List<RexLiteral> values : head.tuples ) collector.add( EnumerableUtil.createTupleFrom( values ) ); collector.close(); }