/** * This method deserializes the Base64 encoded String into an Object instance. * * @param string * @return an Object */ public static <T> T deserializeBase64( String string, Configuration conf, Class<T> type ) throws IOException { return deserializeBase64( string, conf, type, true ); }
public static Configuration removePropertiesFrom( Configuration jobConf, String... keys ) { Map<Object, Object> properties = createProperties( jobConf ); for( String key : keys ) properties.remove( key ); return copyConfiguration( properties, new JobConf() ); }
public static void initLog4j( JobConf configuration ) { initLog4j( (Configuration) configuration ); }
public static JobConf createJobConf( Map<Object, Object> properties, JobConf defaultJobconf ) { JobConf jobConf = defaultJobconf == null ? new JobConf() : copyJobConf( defaultJobconf ); if( properties == null ) return jobConf; return copyConfiguration( properties, jobConf ); }
HadoopUtil.initLog4j( jobConf ); mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" ); flowNode = deserializeBase64( mapNodeState, jobConf, BaseFlowNode.class );
Map<String, Path> remotePaths = new HashMap<String, Path>(); HadoopUtil.resolvePaths( config, classpath, null, null, localPaths, remotePaths ); LocalFileSystem localFS = HadoopUtil.getLocalFS( config ); FileSystem defaultFS = HadoopUtil.getDefaultFS( config ); return HadoopUtil.getCommonPaths( localPaths, remotePaths );
throw new CascadingException( "mapred.input.format.class is required, should be set in source Scheme#sourceConfInit" ); configs.add( HadoopUtil.getConfig( toJob, fromJob ) ); Collections.addAll( allPaths, FileInputFormat.getInputPaths( fromJob ) ); isLocal = HadoopUtil.isLocal( fromJob ); toJob.set( "cascading.multiinputformats", HadoopUtil.serializeBase64( configs, toJob, true ) ); HadoopUtil.setLocal( toJob );
@Override protected boolean isRemoteExecution() { return !HadoopUtil.isLocal( jobConfiguration ); }
@Override public Map<Object, Object> getConfigAsProperties() { return HadoopUtil.createProperties( getConfig() ); }
@Override public <C> C copyConfig( C config ) { return HadoopUtil.copyJobConf( config ); }
public static void resolvePaths( Configuration config, Collection<String> classpath, String remoteRoot, String resourceSubPath, Map<String, Path> localPaths, Map<String, Path> remotePaths ) FileSystem defaultFS = getDefaultFS( config ); FileSystem localFS = getLocalFS( config ); if( !exists( localFS, localPath ) ) throw new FlowException( "path not found: " + localPath ); remotePaths.put( name, defaultFS.makeQualified( new Path( remoteRootPath, path.getName() ) ) ); else if( localFS.equals( getFileSystem( config, path ) ) ) if( !exists( localFS, path ) ) throw new FlowException( "path not found: " + path ); if( !exists( defaultFS, path ) ) throw new FlowException( "path not found: " + path );
@Override public PlatformInfo getPlatformInfo() { return HadoopUtil.getPlatformInfo( DAG.class, null, "Tez" ); }
Map<Path, Path> copyPaths = getCopyPaths( config, commonPaths ); // tests remote file existence or if stale LocalFileSystem localFS = getLocalFS( config ); FileSystem remoteFS = getDefaultFS( config );
FileSystem remoteFS = getDefaultFS( config ); FileSystem localFS = getLocalFS( config );
@Override public void initialize( FlowConnector flowConnector, Map<Object, Object> properties ) { super.initialize( flowConnector, properties ); defaultJobConf = HadoopUtil.createJobConf( properties, createJobConf( properties ) ); checkPlatform( defaultJobConf ); intermediateSchemeClass = flowConnector.getIntermediateSchemeClass( properties ); Class type = AppProps.getApplicationJarClass( properties ); if( defaultJobConf.getJar() == null && type != null ) defaultJobConf.setJarByClass( type ); String path = AppProps.getApplicationJarPath( properties ); if( defaultJobConf.getJar() == null && path != null ) defaultJobConf.setJar( path ); if( defaultJobConf.getJar() == null ) defaultJobConf.setJarByClass( HadoopUtil.findMainClass( HadoopPlanner.class ) ); AppProps.setApplicationJarPath( properties, defaultJobConf.getJar() ); LOG.info( "using application jar: {}", defaultJobConf.getJar() ); }
@Override public <C> Map<String, String> diffConfigIntoMap( C defaultConfig, C updatedConfig ) { return HadoopUtil.getConfig( (Configuration) defaultConfig, (Configuration) updatedConfig ); }
public static JobConf createJobConf( Map<Object, Object> properties ) { return createJobConf( properties, null ); }
HadoopUtil.initLog4j( jobConf ); mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" ); flowNode = deserializeBase64( mapNodeState, jobConf, BaseFlowNode.class );
Map<String, Path> remotePaths = new HashMap<String, Path>(); HadoopUtil.resolvePaths( config, classpath, null, null, localPaths, remotePaths ); LocalFileSystem localFS = HadoopUtil.getLocalFS( config ); FileSystem defaultFS = HadoopUtil.getDefaultFS( config ); return HadoopUtil.getCommonPaths( localPaths, remotePaths );