/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldNames The field names in the table. */ public static void setOutput(JobConf job, String tableName, String... fieldNames) { if(fieldNames.length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldNames(fieldNames); } else { if(fieldNames.length > 0) setOutput(job, tableName, fieldNames.length); else throw new IllegalArgumentException("Field names must be greater than 0"); } }
public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); }
public void testSetOutput() throws IOException { JobConf job = new JobConf(); DBOutputFormat.setOutput(job, "hadoop_output", fieldNames); DBConfiguration dbConf = new DBConfiguration(job); String actual = format.constructQuery(dbConf.getOutputTableName() , dbConf.getOutputFieldNames()); assertEquals(expected, actual); job = new JobConf(); dbConf = new DBConfiguration(job); DBOutputFormat.setOutput(job, "hadoop_output", nullFieldNames.length); assertNull(dbConf.getOutputFieldNames()); assertEquals(nullFieldNames.length, dbConf.getOutputFieldCount()); actual = format.constructQuery(dbConf.getOutputTableName() , new String[dbConf.getOutputFieldCount()]); assertEquals(nullExpected, actual); }
public void testSetOutput() throws IOException { JobConf job = new JobConf(); DBOutputFormat.setOutput(job, "hadoop_output", fieldNames); DBConfiguration dbConf = new DBConfiguration(job); String actual = format.constructQuery(dbConf.getOutputTableName() , dbConf.getOutputFieldNames()); assertEquals(expected, actual); job = new JobConf(); dbConf = new DBConfiguration(job); DBOutputFormat.setOutput(job, "hadoop_output", nullFieldNames.length); assertNull(dbConf.getOutputFieldNames()); assertEquals(nullFieldNames.length, dbConf.getOutputFieldCount()); actual = format.constructQuery(dbConf.getOutputTableName() , new String[dbConf.getOutputFieldCount()]); assertEquals(nullExpected, actual); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldNames The field names in the table. */ public static void setOutput(JobConf job, String tableName, String... fieldNames) { if(fieldNames.length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldNames(fieldNames); } else { if(fieldNames.length > 0) setOutput(job, tableName, fieldNames.length); else throw new IllegalArgumentException("Field names must be greater than 0"); } }
/** {@inheritDoc} */ public RecordWriter<K, V> getRecordWriter(FileSystem filesystem, JobConf job, String name, Progressable progress) throws IOException { DBConfiguration dbConf = new DBConfiguration(job); String tableName = dbConf.getOutputTableName(); String[] fieldNames = dbConf.getOutputFieldNames(); try { Connection connection = dbConf.getConnection(); PreparedStatement statement = null; statement = connection.prepareStatement(constructQuery(tableName, fieldNames)); return new DBRecordWriter(connection, statement); } catch (Exception ex) { throw new IOException(ex.getMessage()); } }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** {@inheritDoc} */ public RecordWriter<K, V> getRecordWriter(FileSystem filesystem, JobConf job, String name, Progressable progress) throws IOException { DBConfiguration dbConf = new DBConfiguration(job); String tableName = dbConf.getOutputTableName(); String[] fieldNames = dbConf.getOutputFieldNames(); try { Connection connection = dbConf.getConnection(); PreparedStatement statement = null; statement = connection.prepareStatement(constructQuery(tableName, fieldNames)); return new DBRecordWriter(connection, statement); } catch (Exception ex) { throw new IOException(ex.getMessage()); } }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldNames The field names in the table. */ public static void setOutput(JobConf job, String tableName, String... fieldNames) { if(fieldNames.length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldNames(fieldNames); } else { if(fieldNames.length > 0) setOutput(job, tableName, fieldNames.length); else throw new IllegalArgumentException("Field names must be greater than 0"); } }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldNames The field names in the table. */ public static void setOutput(JobConf job, String tableName, String... fieldNames) { if(fieldNames.length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldNames(fieldNames); } else { if(fieldNames.length > 0) setOutput(job, tableName, fieldNames.length); else throw new IllegalArgumentException("Field names must be greater than 0"); } }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldNames The field names in the table. */ public static void setOutput(JobConf job, String tableName, String... fieldNames) { if(fieldNames.length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldNames(fieldNames); } else { if(fieldNames.length > 0) setOutput(job, tableName, fieldNames.length); else throw new IllegalArgumentException("Field names must be greater than 0"); } }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
public static void runJob(String mysqlJar, String output) throws Exception { Configuration conf = new Configuration(); JobHelper.addJarForJob(conf, mysqlJar); DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/sqoop_test" + "?user=hip_sqoop_user&password=password"); JobConf job = new JobConf(conf); job.setJarByClass(DBExportMapReduce.class); Path outputPath = new Path(output); outputPath.getFileSystem(job).delete(outputPath, true); job.set(AvroJob.INPUT_SCHEMA, Stock.SCHEMA$.toString()); job.setInputFormat(AvroInputFormat.class); job.setInputFormat(DBInputFormat.class); job.setOutputFormat(DBOutputFormat.class); AvroJob.setOutputSchema(job, Stock.SCHEMA$); job.setMapperClass(Map.class); job.setMapOutputKeyClass(StockRecord.class); job.setMapOutputValueClass(NullWritable.class); job.setOutputKeyClass(StockRecord.class); job.setOutputValueClass(NullWritable.class); job.setNumReduceTasks(4); DBOutputFormat.setOutput( job, "stocks_export", StockRecord.fields); JobClient.runJob(job); }
DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);