private void testCommonSplitterTypes( DataDrivenDBInputFormat<NullDBWritable> format) { assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL) .getClass()); assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC) .getClass()); assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN) .getClass()); assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT) .getClass()); assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER) .getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE) .getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass()); assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT) .getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR) .getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass()); assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR) .getClass()); assertNull(format.getSplitter(Types.BINARY));
@Test(timeout = 10000) public void testDataDrivenDBInputFormat() throws Exception { JobContext jobContext = mock(JobContext.class); Configuration configuration = new Configuration(); configuration.setInt(MRJobConfig.NUM_MAPS, 1); when(jobContext.getConfiguration()).thenReturn(configuration); DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>(); List<InputSplit> splits = format.getSplits(jobContext); assertEquals(1, splits.size()); DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0); assertEquals("1=1", split.getLowerClause()); assertEquals("1=1", split.getUpperClause()); // 2 configuration.setInt(MRJobConfig.NUM_MAPS, 2); DataDrivenDBInputFormat.setBoundingQuery(configuration, "query"); assertEquals("query", configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY)); Job job = mock(Job.class); when(job.getConfiguration()).thenReturn(configuration); DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query", "Bounding Query"); assertEquals("Bounding Query", configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY)); }
/** * test splitters from DataDrivenDBInputFormat. For different data types may * be different splitter */ @Test(timeout = 10000) public void testDataDrivenDBInputFormatSplitter() { DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>(); testCommonSplitterTypes(format); assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP) .getClass()); assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass()); assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass()); }
Connection connection = getConnection(); try { statement = connection.createStatement(); results = statement.executeQuery(getBoundingValsQuery()); results.next(); DBSplitter splitter = getSplitter(sqlDataType); if (null == splitter) { throw new IOException("Unknown SQL data type: " + sqlDataType); return splitter.split(job.getConfiguration(), results, getDBConf().getInputOrderBy()); } catch (SQLException e) { throw new IOException(e.getMessage()); closeConnection(); } catch (SQLException se) { LOG.debug("SQLException committing split transaction: " + se.toString());
protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { DBConfiguration dbConf = getDBConf(); @SuppressWarnings("unchecked") Class<T> inputClass = (Class<T>) (dbConf.getInputClass()); String dbProductName = getDBProductName(); LOG.debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.startsWith("MYSQL")) { // use MySQL-specific db reader. return new MySQLDataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName()); } else { // Generic reader. return new DataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName(), dbProductName); } } catch (SQLException ex) { throw new IOException(ex.getMessage()); } }
protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { DBConfiguration dbConf = getDBConf(); @SuppressWarnings("unchecked") Class<T> inputClass = (Class<T>) (dbConf.getInputClass()); String dbProductName = getDBProductName(); LOG.debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.startsWith("MYSQL")) { // use MySQL-specific db reader. return new MySQLDataDrivenDBRecordReader<T>(split, inputClass, conf, getConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName()); } else { // Generic reader. return new DataDrivenDBRecordReader<T>(split, inputClass, conf, getConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName(), dbProductName); } } catch (SQLException ex) { throw new IOException(ex.getMessage()); } }
/** * @return a query which returns the minimum and maximum values for * the order-by column. * * The min value should be in the first column, and the * max value should be in the second column of the results. */ protected String getBoundingValsQuery() { // If the user has provided a query, use that instead. String userQuery = getDBConf().getInputBoundingQuery(); if (null != userQuery) { return userQuery; } // Auto-generate one based on the table name we've been provided with. StringBuilder query = new StringBuilder(); String splitCol = getDBConf().getInputOrderBy(); query.append("SELECT MIN(").append(splitCol).append("), "); query.append("MAX(").append(splitCol).append(") FROM "); query.append(getDBConf().getInputTableName()); String conditions = getDBConf().getInputConditions(); if (null != conditions) { query.append(" WHERE ( " + conditions + " )"); } return query.toString(); }
DBConfiguration.configureDB(job.getConfiguration(), DRIVER_CLASS, DB_URL, null, null); DataDrivenDBInputFormat.setInput(job, DateCol.class, DATE_TABLE, null, COL, COL);
statement = connection.createStatement(); results = statement.executeQuery(getBoundingValsQuery()); results.next(); DBSplitter splitter = getSplitter(sqlDataType); if (null == splitter) { throw new IOException("Unknown SQL data type: " + sqlDataType); return splitter.split(job.getConfiguration(), results, getDBConf().getInputOrderBy()); } catch (SQLException e) { throw new IOException(e.getMessage()); closeConnection(); } catch (SQLException se) { LOG.debug("SQLException committing split transaction: " + se.toString());
protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { DBConfiguration dbConf = getDBConf(); @SuppressWarnings("unchecked") Class<T> inputClass = (Class<T>) (dbConf.getInputClass()); String dbProductName = getDBProductName(); LOG.debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.startsWith("MYSQL")) { // use MySQL-specific db reader. return new MySQLDataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName()); } else { // Generic reader. return new DataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName(), dbProductName); } } catch (SQLException ex) { throw new IOException(ex.getMessage()); } }
/** * @return a query which returns the minimum and maximum values for * the order-by column. * * The min value should be in the first column, and the * max value should be in the second column of the results. */ protected String getBoundingValsQuery() { // If the user has provided a query, use that instead. String userQuery = getDBConf().getInputBoundingQuery(); if (null != userQuery) { return userQuery; } // Auto-generate one based on the table name we've been provided with. StringBuilder query = new StringBuilder(); String splitCol = getDBConf().getInputOrderBy(); query.append("SELECT MIN(").append(splitCol).append("), "); query.append("MAX(").append(splitCol).append(") FROM "); query.append(getDBConf().getInputTableName()); String conditions = getDBConf().getInputConditions(); if (null != conditions) { query.append(" WHERE ( " + conditions + " )"); } return query.toString(); }
DBConfiguration.configureDB(job.getConfiguration(), DRIVER_CLASS, DB_URL, null, null); DataDrivenDBInputFormat.setInput(job, DateCol.class, DATE_TABLE, null, COL, COL);
statement = connection.createStatement(); results = statement.executeQuery(getBoundingValsQuery()); results.next(); DBSplitter splitter = getSplitter(sqlDataType); if (null == splitter) { throw new IOException("Unknown SQL data type: " + sqlDataType); return splitter.split(job.getConfiguration(), results, getDBConf().getInputOrderBy()); } catch (SQLException e) { throw new IOException(e.getMessage()); closeConnection(); } catch (SQLException se) { LOG.debug("SQLException committing split transaction: " + se.toString());
protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { DBConfiguration dbConf = getDBConf(); @SuppressWarnings("unchecked") Class<T> inputClass = (Class<T>) (dbConf.getInputClass()); String dbProductName = getDBProductName(); LOG.debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.startsWith("MYSQL")) { // use MySQL-specific db reader. return new MySQLDataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName()); } else { // Generic reader. return new DataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName(), dbProductName); } } catch (SQLException ex) { throw new IOException(ex.getMessage()); } }
/** * @return the DBSplitter implementation to use to divide the table/query into InputSplits. */ @Override protected DBSplitter getSplitter(int sqlDataType) { switch (sqlDataType) { case Types.DATE: case Types.TIME: case Types.TIMESTAMP: return new OracleDateSplitter(); default: return super.getSplitter(sqlDataType); } }
/** * @return a query which returns the minimum and maximum values for * the order-by column. * * The min value should be in the first column, and the * max value should be in the second column of the results. */ protected String getBoundingValsQuery() { // If the user has provided a query, use that instead. String userQuery = getDBConf().getInputBoundingQuery(); if (null != userQuery) { return userQuery; } // Auto-generate one based on the table name we've been provided with. StringBuilder query = new StringBuilder(); String splitCol = getDBConf().getInputOrderBy(); query.append("SELECT MIN(").append(splitCol).append("), "); query.append("MAX(").append(splitCol).append(") FROM "); query.append(getDBConf().getInputTableName()); String conditions = getDBConf().getInputConditions(); if (null != conditions) { query.append(" WHERE ( " + conditions + " )"); } return query.toString(); }
statement = connection.createStatement(); results = statement.executeQuery(getBoundingValsQuery()); results.next(); DBSplitter splitter = getSplitter(sqlDataType); if (null == splitter) { throw new IOException("Unknown SQL data type: " + sqlDataType); return splitter.split(job.getConfiguration(), results, getDBConf().getInputOrderBy()); } catch (SQLException e) { throw new IOException(e.getMessage()); closeConnection(); } catch (SQLException se) { LOG.debug("SQLException committing split transaction: " + se.toString());
protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { DBConfiguration dbConf = getDBConf(); @SuppressWarnings("unchecked") Class<T> inputClass = (Class<T>) (dbConf.getInputClass()); String dbProductName = getDBProductName(); LOG.debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.startsWith("MYSQL")) { // use MySQL-specific db reader. return new MySQLDataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName()); } else { // Generic reader. return new DataDrivenDBRecordReader<T>(split, inputClass, conf, createConnection(), dbConf, dbConf.getInputConditions(), dbConf.getInputFieldNames(), dbConf.getInputTableName(), dbProductName); } } catch (SQLException ex) { throw new IOException(ex.getMessage()); } }
/** * @return the DBSplitter implementation to use to divide the table/query into InputSplits. */ @Override protected DBSplitter getSplitter(int sqlDataType) { switch (sqlDataType) { case Types.DATE: case Types.TIME: case Types.TIMESTAMP: return new OracleDateSplitter(); default: return super.getSplitter(sqlDataType); } }
/** * @return a query which returns the minimum and maximum values for * the order-by column. * * The min value should be in the first column, and the * max value should be in the second column of the results. */ protected String getBoundingValsQuery() { // If the user has provided a query, use that instead. String userQuery = getDBConf().getInputBoundingQuery(); if (null != userQuery) { return userQuery; } // Auto-generate one based on the table name we've been provided with. StringBuilder query = new StringBuilder(); String splitCol = getDBConf().getInputOrderBy(); query.append("SELECT MIN(").append(splitCol).append("), "); query.append("MAX(").append(splitCol).append(") FROM "); query.append(getDBConf().getInputTableName()); String conditions = getDBConf().getInputConditions(); if (null != conditions) { query.append(" WHERE ( " + conditions + " )"); } return query.toString(); }