@Override public Connection connect(String arg0, Properties arg1) throws SQLException { return getConnection(); }
@Override public void write(NullWritable key, T record) throws IOException { try { record.write(pstmt); numRecords++; pstmt.executeUpdate(); if (numRecords % batchSize == 0) { LOG.debug("Commit called on a batch of size : " + batchSize); conn.commit(); } } catch (SQLException e) { throw new IOException("Exception while writing to table.", e); } }
/** * test generate sql script for OracleDBRecordReader. */ @Test(timeout = 20000) public void testOracleDBRecordReader() throws Exception { DBInputSplit splitter = new DBInputSplit(1, 10); Configuration configuration = new Configuration(); Connection connect = DriverForTest.getConnection(); DBConfiguration dbConfiguration = new DBConfiguration(configuration); dbConfiguration.setInputOrderBy("Order"); String[] fields = { "f1", "f2" }; OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>( splitter, NullDBWritable.class, configuration, connect, dbConfiguration, "condition", fields, "table"); assertEquals( "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1", recorder.getSelectQuery()); }
@Override public DBConfiguration getDBConf() { String[] names = { "field1", "field2" }; DBConfiguration result = mock(DBConfiguration.class); when(result.getInputConditions()).thenReturn("conditions"); when(result.getInputFieldNames()).thenReturn(names); when(result.getInputTableName()).thenReturn("table"); return result; }
@Test(timeout=2000) public void testTextSplitter() throws Exception{ TextSplitter splitter = new TextSplitter(); ResultSet result = mock(ResultSet.class); List<InputSplit> splits = splitter.split(configuration, result, "column"); assertSplits(new String[] {"column IS NULL column IS NULL"}, splits); when(result.getString(1)).thenReturn("result1"); when(result.getString(2)).thenReturn("result2"); splits = splitter.split(configuration, result, "column1"); assertSplits(new String[] {"column1 >= 'result1' column1 < 'result1.'", "column1 >= 'result1' column1 <= 'result2'"}, splits); }
@Test(timeout=2000) public void testIntegerSplitter() throws Exception{ IntegerSplitter splitter = new IntegerSplitter(); ResultSet result = mock(ResultSet.class); List<InputSplit> splits = splitter.split(configuration, result, "column"); assertSplits(new String[] {".*column IS NULL"}, splits); when(result.getString(1)).thenReturn("result1"); when(result.getString(2)).thenReturn("result2"); when(result.getLong(1)).thenReturn(8L); when(result.getLong(2)).thenReturn(19L); splits = splitter.split(configuration, result, "column1"); assertSplits(new String[] {"column1 >= 8 column1 < 13", "column1 >= 13 column1 < 18", "column1 >= 18 column1 <= 19"}, splits); }
@Test(timeout = 10000) public void testDataDrivenDBInputFormat() throws Exception { JobContext jobContext = mock(JobContext.class); Configuration configuration = new Configuration(); configuration.setInt(MRJobConfig.NUM_MAPS, 1); when(jobContext.getConfiguration()).thenReturn(configuration); DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>(); List<InputSplit> splits = format.getSplits(jobContext); assertEquals(1, splits.size()); DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0); assertEquals("1=1", split.getLowerClause()); assertEquals("1=1", split.getUpperClause()); // 2 configuration.setInt(MRJobConfig.NUM_MAPS, 2); DataDrivenDBInputFormat.setBoundingQuery(configuration, "query"); assertEquals("query", configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY)); Job job = mock(Job.class); when(job.getConfiguration()).thenReturn(configuration); DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query", "Bounding Query"); assertEquals("Bounding Query", configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY)); }
@Test(timeout=2000) public void testBigDecimalSplitter() throws Exception{ BigDecimalSplitter splitter = new BigDecimalSplitter(); ResultSet result = mock(ResultSet.class); List<InputSplit> splits = splitter.split(configuration, result, "column"); assertSplits(new String[] {".*column IS NULL"}, splits); when(result.getString(1)).thenReturn("result1"); when(result.getString(2)).thenReturn("result2"); when(result.getBigDecimal(1)).thenReturn(new BigDecimal(10)); when(result.getBigDecimal(2)).thenReturn(new BigDecimal(12)); splits = splitter.split(configuration, result, "column1"); assertSplits(new String[] {"column1 >= 10 column1 < 11", "column1 >= 11 column1 <= 12"}, splits); }
private void assertSplits(String[] expectedSplitRE, List<InputSplit> splits) throws IOException { assertEquals(expectedSplitRE.length, splits.size()); for (int i = 0; i < expectedSplitRE.length; i++) { DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(i); String actualExpr = split.getLowerClause() + " " + split.getUpperClause(); assertTrue("Split #" + (i+1) + " expression is wrong." + " Expected " + expectedSplitRE[i] + " Actual " + actualExpr, Pattern.matches(expectedSplitRE[i], actualExpr)); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (key == null) { key = NullWritable.get(); } if (value == null) { value = ReflectionUtils.newInstance(inputClass, this.configuration); } Preconditions.checkNotNull(this.resultSet); try { if(!resultSet.next()) { return false; } value.readFields(resultSet); return true; } catch (SQLException e) { LOG.error(String.format(" Error [%s] occurred while iterating over the resultset. ",e.getMessage())); throw new RuntimeException(e); } }
@Test(timeout=2000) public void testFloatSplitter() throws Exception{ FloatSplitter splitter = new FloatSplitter(); ResultSet results = mock(ResultSet.class); List<InputSplit> splits = splitter.split(configuration, results, "column"); assertSplits(new String[] {".*column IS NULL"}, splits); when(results.getString(1)).thenReturn("result1"); when(results.getString(2)).thenReturn("result2"); when(results.getDouble(1)).thenReturn(5.0); when(results.getDouble(2)).thenReturn(7.0); splits = splitter.split(configuration, results, "column1"); assertSplits(new String[] {"column1 >= 5.0 column1 < 6.0", "column1 >= 6.0 column1 <= 7.0"}, splits); }
/** * test splitters from DataDrivenDBInputFormat. For different data types may * be different splitter */ @Test(timeout = 10000) public void testDataDrivenDBInputFormatSplitter() { DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>(); testCommonSplitterTypes(format); assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP) .getClass()); assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass()); assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass()); }
@Test(timeout=2000) public void testBooleanSplitter() throws Exception{ BooleanSplitter splitter = new BooleanSplitter(); ResultSet result = mock(ResultSet.class); when(result.getString(1)).thenReturn("result1"); List<InputSplit> splits=splitter.split(configuration, result, "column"); assertSplits(new String[] {"column = FALSE column = FALSE", "column IS NULL column IS NULL"}, splits); when(result.getString(1)).thenReturn("result1"); when(result.getString(2)).thenReturn("result2"); when(result.getBoolean(1)).thenReturn(true); when(result.getBoolean(2)).thenReturn(false); splits=splitter.split(configuration, result, "column"); assertEquals(0, splits.size()); when(result.getString(1)).thenReturn("result1"); when(result.getString(2)).thenReturn("result2"); when(result.getBoolean(1)).thenReturn(false); when(result.getBoolean(2)).thenReturn(true); splits = splitter.split(configuration, result, "column"); assertSplits(new String[] { "column = FALSE column = FALSE", ".*column = TRUE"}, splits); }
private void testCommonSplitterTypes( DataDrivenDBInputFormat<NullDBWritable> format) { assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL) .getClass()); assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC) .getClass()); assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN) .getClass()); assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER) .getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE) .getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT) .getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR) .getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR) .getClass()); assertNull(format.getSplitter(Types.BINARY));
@Test(timeout = 10000) public void testOracleDataDrivenDBInputFormat() throws Exception { OracleDataDrivenDBInputFormat<NullDBWritable> format = new OracleDataDrivenDBInputFormatForTest(); testCommonSplitterTypes(format); assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP) .getClass()); assertEquals(OracleDateSplitter.class, format.getSplitter(Types.DATE).getClass()); assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIME).getClass()); }
private void setupDriver(JobConf configuration) throws Exception { configuration.set(DBConfiguration.URL_PROPERTY, "testUrl"); DriverManager.registerDriver(new DriverForTest()); configuration.set(DBConfiguration.DRIVER_CLASS_PROPERTY, DriverForTest.class.getCanonicalName()); }
@Override public boolean next(WritableComparable key, T value) throws IOException { try { if (!resultSet.next()) { return false; } value.readFields(resultSet); if (isTransactional) { ((PhoenixResultWritable) value).readPrimaryKey((PhoenixRowKey) key); } ++readCount; if (LOG.isTraceEnabled()) { LOG.trace("Result[" + readCount + "] : " + ((PhoenixResultWritable) value) .getResultMap()); } return true; } catch (SQLException e) { LOG.error(String.format(" Error [%s] occurred while iterating over the resultset. ", e.getMessage())); throw new RuntimeException(e); } }
@Override public void write(NullWritable n, T record) throws IOException, InterruptedException { try { record.write(statement); numRecords++; statement.execute(); if (numRecords % batchSize == 0) { LOG.debug("commit called on a batch of size : " + batchSize); conn.commit(); } } catch (SQLException e) { throw new RuntimeException("Exception while committing to database.", e); } }
@Override public Connection getConnection() { return DriverForTest.getConnection(); }
/** * * test DBRecordReader. This reader should creates keys, values, know about position.. */ @SuppressWarnings("unchecked") @Test (timeout = 5000) public void testDBRecordReader() throws Exception { JobConf job = mock(JobConf.class); DBConfiguration dbConfig = mock(DBConfiguration.class); String[] fields = { "field1", "filed2" }; @SuppressWarnings("rawtypes") DBRecordReader reader = new DBInputFormat<NullDBWritable>().new DBRecordReader( new DBInputSplit(), NullDBWritable.class, job, DriverForTest.getConnection(), dbConfig, "condition", fields, "table"); LongWritable key = reader.createKey(); assertEquals(0, key.get()); DBWritable value = reader.createValue(); assertEquals( "org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable", value .getClass().getName()); assertEquals(0, reader.getPos()); assertFalse(reader.next(key, value)); }