protected void initConfig( Map<Object, Object> properties, JobConf parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; jobConf = HadoopUtil.copyJobConf( parentConfig ); // prevent local values from being shared jobConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); jobConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); syncPaths = HadoopMRUtil.addToClassPath( jobConf, getClassPath() ); }
mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" );
private List<ElementGraph> createNodeElementGraphs( JobConf jobConf ) boolean hasReducer = HadoopMRUtil.hasReducer( jobConf );
conf.set( "cascading.flow.step.node.map.path", HadoopMRUtil.writeStateToDistCache( conf, getID(), "map", mapState ) ); conf.set( "cascading.flow.step.node.reduce.path", HadoopMRUtil.writeStateToDistCache( conf, getID(), "reduce", reduceState ) );
mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" );
private List<ElementGraph> createNodeElementGraphs( JobConf jobConf ) boolean hasReducer = HadoopMRUtil.hasReducer( jobConf );
conf.set( "cascading.flow.step.node.map.path", HadoopMRUtil.writeStateToDistCache( conf, getID(), "map", mapState ) ); conf.set( "cascading.flow.step.node.reduce.path", HadoopMRUtil.writeStateToDistCache( conf, getID(), "reduce", reduceState ) );
reduceNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "reduce" );
protected void initConfig( Map<Object, Object> properties, JobConf parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; jobConf = HadoopUtil.copyJobConf( parentConfig ); // prevent local values from being shared jobConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); jobConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); syncPaths = HadoopMRUtil.addToClassPath( jobConf, getClassPath() ); }
reduceNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "reduce" );