/** * Configures the Flink program for this step */ public Configuration createInitializedConfig( FlowProcess<Configuration> flowProcess, Configuration parentConfig ) { this.env.getConfig().registerKryoType(Tuple.class); Configuration config = parentConfig == null ? new JobConf() : HadoopUtil.copyJobConf( parentConfig ); config.set( "cascading.flow.step.num", Integer.toString( getOrdinal() ) ); HadoopUtil.setIsInflow(config); this.setConfig(config); return config; }
private void writeToHadoopPartitionTap(Tap<?, ?, ?> tap) throws IOException { @SuppressWarnings("unchecked") BasePartitionTap<JobConf, ?, ?> hadoopTap = (BasePartitionTap<JobConf, ?, ?>) tap; JobConf conf = new JobConf(); // Avoids deletion of results when using a partition tap (close() will delete the _temporary before the copy has // been done if not in a flow) HadoopUtil.setIsInflow(conf); HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf); hadoopTap.sinkConfInit(flowProcess, conf); TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess); for (TupleEntry tuple : data.asTupleEntryList()) { collector.add(tuple); } collector.close(); // We need to clean up the '_temporary' folder BasePartitionTap<JobConf, ?, ?> partitionTap = hadoopTap; @SuppressWarnings("unchecked") String basePath = partitionTap.getParent().getFullIdentifier(flowProcess); deleteTemporaryPath(new Path(basePath), FileSystem.get(conf)); }
HadoopUtil.setIsInflow( conf ); // must be called after all taps configurations have been retrieved
HadoopUtil.setIsInflow( conf ); // must be called after all taps configurations have been retrieved
conf.set( "cascading.flow.step.num", Integer.toString( getOrdinal() ) ); HadoopUtil.setIsInflow( conf );
conf.set( "cascading.flow.step.num", Integer.toString( getOrdinal() ) ); HadoopUtil.setIsInflow( conf );