@Override protected void configure( Configuration conf ) throws Exception { callVisitors(); setMapRunnerClass( hadoopShim.getPentahoMapReduceMapRunnerClass().getCanonicalName() ); conf.set( TRANSFORMATION_MAP_XML, mapperTransformationXml ); conf.set( TRANSFORMATION_MAP_INPUT_STEPNAME, mapperInputStep ); conf.set( TRANSFORMATION_MAP_OUTPUT_STEPNAME, mapperOutputStep ); if ( combinerTransformationXml != null ) { conf.set( TRANSFORMATION_COMBINER_XML, combinerTransformationXml ); conf.set( TRANSFORMATION_COMBINER_INPUT_STEPNAME, combinerInputStep ); conf.set( TRANSFORMATION_COMBINER_OUTPUT_STEPNAME, combinerOutputStep ); setCombinerClass( hadoopShim.getPentahoMapReduceCombinerClass().getCanonicalName() ); } if ( reducerTransformationXml != null ) { conf.set( TRANSFORMATION_REDUCE_XML, reducerTransformationXml ); conf.set( TRANSFORMATION_REDUCE_INPUT_STEPNAME, reducerInputStep ); conf.set( TRANSFORMATION_REDUCE_OUTPUT_STEPNAME, reducerOutputStep ); setReducerClass( hadoopShim.getPentahoMapReduceReducerClass().getCanonicalName() ); } conf.setJarByClass( hadoopShim.getPentahoMapReduceMapRunnerClass() ); conf.set( LOG_LEVEL, logLevel.toString() ); configureVariableSpace( conf ); super.configure( conf ); }
@Override public void configureConnectionInformation( String namenodeHost, String namenodePort, String jobtrackerHost, String jobtrackerPort, Configuration conf, List<String> logMessages ) throws Exception { if ( namenodeHost == null || namenodeHost.trim().length() == 0 ) { throw new Exception( "No hdfs host specified!" ); } if ( jobtrackerHost == null || jobtrackerHost.trim().length() == 0 ) { throw new Exception( "No job tracker host specified!" ); } if ( namenodePort != null && namenodePort.trim().length() != 0 && !"-1".equals( namenodePort.trim() ) ) { namenodePort = ":" + namenodePort; } else { // it's been realized that this is pretty fine to have // NameNode URL w/o port: e.g. HA mode (BAD-358) namenodePort = ""; logMessages.add( "No hdfs port specified - HA? " ); } if ( jobtrackerPort == null || jobtrackerPort.trim().length() == 0 ) { jobtrackerPort = getDefaultJobtrackerPort(); logMessages.add( "No job tracker port specified - using default: " + jobtrackerPort ); } String fsDefaultName = "hdfs://" + namenodeHost + namenodePort; String jobTracker = jobtrackerHost + ":" + jobtrackerPort; conf.set( "fs.default.name", fsDefaultName ); conf.set( "mapred.job.tracker", jobTracker ); }
String value = stringStringEntry.getValue(); if ( key != null && !"".equals( key ) && value != null && !"".equals( value ) ) { conf.set( key, value );
conf.set( "fs.maprfs.impl", MapRFileProvider.FS_MAPR_IMPL );
conf.set( "fs.maprfs.impl", MapRFileProvider.FS_MAPR_IMPL );
@Override public int runTool( List<String> args, Properties properties ) { Configuration configuration = hadoopShim.createConfiguration(); for ( Map.Entry<String, String> entry : Maps.fromProperties( properties ).entrySet() ) { configuration.set( entry.getKey(), entry.getValue() ); } try { // Make sure Sqoop throws exceptions instead of returning a status of 1 System.setProperty( SQOOP_THROW_ON_ERROR, Boolean.toString( true ) ); configureShim( configuration ); return sqoopShim.runTool( args.toArray( new String[args.size()] ), configuration ); } catch ( Exception e ) { LOGGER.error( "Error executing sqoop", e ); return 1; } }
conf.set( MAPREDUCE_APPLICATION_CLASSPATH, CLASSES + mapreduceClasspath );