/** * Create a job with the specified name, where the job performs * an import configured with 'jobArgs'. */ private void createJob(String jobName, List<String> jobArgs) { createJob(jobName, jobArgs, newConf()); }
args.add("-m"); args.add("1"); createJob(TABLE_NAME, args, conf); HiveImport.setTestMode(true); String hiveHome = org.apache.sqoop.SqoopOptions.getHiveHomeDefault();
@Test public void testAppendWithTimestamp() throws Exception { // Create a table with data in it; import it. // Then add more data, verify that only the incremental data is pulled. final String TABLE_NAME = "appendTimestamp"; Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows. long importWasBefore = System.currentTimeMillis(); Thread.sleep(50); long rowsAddedTime = System.currentTimeMillis() - 5; assertTrue(rowsAddedTime > importWasBefore); assertTrue(rowsAddedTime < System.currentTimeMillis()); insertIdTimestampRows(TABLE_NAME, 10, 20, new Timestamp(rowsAddedTime)); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
args.add("--merge-key"); args.add("ID"); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10);
@Test public void testAppendWithString() throws Exception { // Create a table with string column in it; // incrementally import it on the string column - it should fail. final String TABLE_NAME = "appendString"; createIdVarcharTable(TABLE_NAME, 10); List<String> args = getArgListForTable(TABLE_NAME, false, true); args.add("--append"); createJob(TABLE_NAME, args); thrown.expect(RuntimeException.class); thrown.reportMissingExceptionWithMessage("Expected incremental import on varchar column to fail"); runJob(TABLE_NAME); }
createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10);
@Test public void testTimestampBoundary() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "boundaryTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 0);
@Test public void testIncrementalAppendTimestamp() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "incrementalAppendTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, true, true); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } @Test
@Test public void testTableNameWithSpecialCharacters() throws Exception { // Table name with special characters to verify proper table name escaping final String TABLE_NAME = "my-table.ext"; createIdTable(TABLE_NAME, 0); // Now add some rows. insertIdRows(TABLE_NAME, 0, 10); List<String> args = getArgListForTable(TABLE_NAME, false, true); createJob("emptyJob", args); runJob("emptyJob"); assertDirOfNumbers(TABLE_NAME, 10); }
@Test public void testEmptyJobLastMod() throws Exception { // Create a job and run an import on an empty table. // Nothing should happen. final String TABLE_NAME = "emptyJobLastMod"; createTimestampTable(TABLE_NAME, 0, null); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob("emptyJobLastMod", args); runJob("emptyJobLastMod"); assertDirOfNumbers(TABLE_NAME, 0); // Running the job a second time should result in // nothing happening, it's still empty. runJob("emptyJobLastMod"); assertDirOfNumbers(TABLE_NAME, 0); }
@Test public void testEmptyJobAppend() throws Exception { // Create a job and run an import on an empty table. // Nothing should happen. final String TABLE_NAME = "emptyJob"; createIdTable(TABLE_NAME, 0); List<String> args = getArgListForTable(TABLE_NAME, false, true); createJob("emptyJob", args); runJob("emptyJob"); assertDirOfNumbers(TABLE_NAME, 0); // Running the job a second time should result in // nothing happening, it's still empty. runJob("emptyJob"); assertDirOfNumbers(TABLE_NAME, 0); }
@Test public void testAppend() throws Exception { // Create a table with data in it; import it. // Then add more data, verify that only the incremental data is pulled. final String TABLE_NAME = "append"; createIdTable(TABLE_NAME, 10); List<String> args = getArgListForTable(TABLE_NAME, false, true); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows. insertIdRows(TABLE_NAME, 10, 20); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
@Test public void testEmptyThenFullJobAppend() throws Exception { // Create an empty table. Import it; nothing happens. // Add some rows. Verify they are appended. final String TABLE_NAME = "emptyThenFull"; createIdTable(TABLE_NAME, 0); List<String> args = getArgListForTable(TABLE_NAME, false, true); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 0); // Now add some rows. insertIdRows(TABLE_NAME, 0, 10); // Running the job a second time should import 10 rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows. insertIdRows(TABLE_NAME, 10, 20); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
@Test public void testEmptyThenFullJobAppendWithQuery() throws Exception { // Create an empty table. Import it; nothing happens. // Add some rows. Verify they are appended. final String TABLE_NAME = "withQuery"; createIdTable(TABLE_NAME, 0); clearDir(TABLE_NAME); final String QUERY = "SELECT id FROM \"withQuery\" WHERE $CONDITIONS"; List<String> args = getArgListForQuery(QUERY, TABLE_NAME, false, true, false); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 0); // Now add some rows. insertIdRows(TABLE_NAME, 0, 10); // Running the job a second time should import 10 rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows. insertIdRows(TABLE_NAME, 10, 20); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }