public static void runJob(String mysqlJar, String output) throws Exception { Configuration conf = new Configuration(); JobHelper.addJarForJob(conf, mysqlJar); DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/sqoop_test" + "?user=hip_sqoop_user&password=password"); JobConf job = new JobConf(conf); job.setJarByClass(DBImportMapReduce.class); Path outputPath = new Path(output); outputPath.getFileSystem(job).delete(outputPath, true); job.setInputFormat(DBInputFormat.class); job.setOutputFormat(AvroOutputFormat.class); AvroJob.setOutputSchema(job, Stock.SCHEMA$); job.set(AvroJob.OUTPUT_CODEC, SnappyCodec.class.getName()); job.setMapperClass(Map.class); job.setNumMapTasks(4); job.setNumReduceTasks(0); job.setMapOutputKeyClass(AvroWrapper.class); job.setMapOutputValueClass(NullWritable.class); job.setOutputKeyClass(AvroWrapper.class); job.setOutputValueClass(NullWritable.class); FileOutputFormat.setOutputPath(job, outputPath); DBInputFormat.setInput( job, StockRecord.class, "select * from stocks", "SELECT COUNT(id) FROM stocks"); JobClient.runJob(job); }
DBInputFormat.setInput(job, AccessRecord.class, "Access", null, "url", AccessFieldNames);