/** * * test DBRecordReader. This reader should creates keys, values, know about position.. */ @SuppressWarnings("unchecked") @Test (timeout = 5000) public void testDBRecordReader() throws Exception { JobConf job = mock(JobConf.class); DBConfiguration dbConfig = mock(DBConfiguration.class); String[] fields = { "field1", "filed2" }; @SuppressWarnings("rawtypes") DBRecordReader reader = new DBInputFormat<NullDBWritable>().new DBRecordReader( new DBInputSplit(), NullDBWritable.class, job, DriverForTest.getConnection(), dbConfig, "condition", fields, "table"); LongWritable key = reader.createKey(); assertEquals(0, key.get()); DBWritable value = reader.createValue(); assertEquals( "org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable", value .getClass().getName()); assertEquals(0, reader.getPos()); assertFalse(reader.next(key, value)); }
/** * test DBInputFormat class. Class should split result for chunks * @throws Exception */ @Test(timeout = 10000) public void testDBInputFormat() throws Exception { JobConf configuration = new JobConf(); setupDriver(configuration); DBInputFormat<NullDBWritable> format = new DBInputFormat<NullDBWritable>(); format.setConf(configuration); format.setConf(configuration); DBInputFormat.DBInputSplit splitter = new DBInputFormat.DBInputSplit(1, 10); Reporter reporter = mock(Reporter.class); RecordReader<LongWritable, NullDBWritable> reader = format.getRecordReader( splitter, configuration, reporter); configuration.setInt(MRJobConfig.NUM_MAPS, 3); InputSplit[] lSplits = format.getSplits(configuration, 3); assertEquals(5, lSplits[0].getLength()); assertEquals(3, lSplits.length); // test reader .Some simple tests assertEquals(LongWritable.class, reader.createKey().getClass()); assertEquals(0, reader.getPos()); assertEquals(0, reader.getProgress(), 0.001); reader.close(); }
/** {@inheritDoc} */ public RecordReader<LongWritable, T> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // wrap the DBRR in a shim class to deal with API differences. return new DBRecordReaderWrapper<T>( (org.apache.hadoop.mapreduce.lib.db.DBRecordReader<T>) createDBRecordReader( (org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit) split, job)); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** * Sets the DB access related fields in the JobConf. * @param job the job * @param driverClass JDBC Driver class name * @param dbUrl JDBC DB access URL. */ public static void configureDB(JobConf job, String driverClass, String dbUrl) { configureDB(job, driverClass, dbUrl, null, null); }
DBInputFormat.setInput(configuration, NullDBWritable.class, "table", "conditions", "orderBy", fieldNames); assertEquals( DBInputFormat.setInput(configuration, NullDBWritable.class, "query", "countQuery"); assertEquals("query", configuration.get(DBConfiguration.INPUT_QUERY, null)); DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl", "user", "password"); assertEquals("driverClass", jConfiguration.get(DBConfiguration.PASSWORD_PROPERTY)); jConfiguration = new JobConf(); DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl"); assertEquals("driverClass", jConfiguration.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
public class Configuration { private DBConfiguration dbConfiguration; public Configuration() { Properties props = readPropertiesFile(); dbConfiguration = new DBConfiguration(props); } public String getConnectionString() { return dbConfiguration.getConnectionString(); } }
public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** {@inheritDoc} */ @SuppressWarnings("unchecked") public RecordReader<LongWritable, T> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // wrap the DBRR in a shim class to deal with API differences. return new DBRecordReaderWrapper<T>( (org.apache.hadoop.mapreduce.lib.db.DBRecordReader<T>) createDBRecordReader( (org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit) split, job)); }
/** * Sets the DB access related fields in the JobConf. * @param job the job * @param driverClass JDBC Driver class name * @param dbUrl JDBC DB access URL. */ public static void configureDB(JobConf job, String driverClass, String dbUrl) { configureDB(job, driverClass, dbUrl, null, null); }
public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** {@inheritDoc} */ public RecordReader<LongWritable, T> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // wrap the DBRR in a shim class to deal with API differences. return new DBRecordReaderWrapper<T>( (org.apache.hadoop.mapreduce.lib.db.DBRecordReader<T>) createDBRecordReader( (org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit) split, job)); }
/** * Sets the DB access related fields in the JobConf. * @param job the job * @param driverClass JDBC Driver class name * @param dbUrl JDBC DB access URL. */ public static void configureDB(JobConf job, String driverClass, String dbUrl) { configureDB(job, driverClass, dbUrl, null, null); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** {@inheritDoc} */ public RecordReader<LongWritable, T> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // wrap the DBRR in a shim class to deal with API differences. return new DBRecordReaderWrapper<T>( (org.apache.hadoop.mapreduce.lib.db.DBRecordReader<T>) createDBRecordReader( (org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit) split, job)); }
/** * Sets the DB access related fields in the JobConf. * @param job the job * @param driverClass JDBC Driver class name * @param dbUrl JDBC DB access URL. */ public static void configureDB(JobConf job, String driverClass, String dbUrl) { configureDB(job, driverClass, dbUrl, null, null); }
/** * Initializes the reduce-part of the job with the appropriate output settings * * @param job The job * @param tableName The table to insert data into * @param fieldCount the number of fields in the table. */ public static void setOutput(JobConf job, String tableName, int fieldCount) { DBConfiguration dbConf = setOutput(job, tableName); dbConf.setOutputFieldCount(fieldCount); }
/** {@inheritDoc} */ public RecordReader<LongWritable, T> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // wrap the DBRR in a shim class to deal with API differences. return new DBRecordReaderWrapper<T>( (org.apache.hadoop.mapreduce.lib.db.DBRecordReader<T>) createDBRecordReader( (org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit) split, job)); }