/** * Create a job with the specified name, where the job performs * an import configured with 'jobArgs'. */ private void createJob(String jobName, List<String> jobArgs) { createJob(jobName, jobArgs, newConf()); }
/** * Run the specified job. */ private void runJob(String jobName) { runJob(jobName, newConf()); }
Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName());
@Test public void testFullLastModifiedImport() throws Exception { // Given a table of rows imported in the past, // see that they are imported. final String TABLE_NAME = "fullLastModified"; Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List<String> args = getArgListForTable(TABLE_NAME, true, false); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(TABLE_NAME, 10); }
@Test public void testNoImportFromTheFuture() throws Exception { // If last-modified dates for writes are serialized to be in the // future w.r.t. an import, do not import these rows. final String TABLE_NAME = "futureLastModified"; Timestamp theFuture = new Timestamp(System.currentTimeMillis() + 1000000); createTimestampTable(TABLE_NAME, 10, theFuture); List<String> args = getArgListForTable(TABLE_NAME, true, false); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(TABLE_NAME, 0); }
@Test public void testLastModifiedImportWithExistingOutputDirectoryFails() throws Exception { final String TABLE_NAME = "failWithExistingOutputDirectory"; createDir(TABLE_NAME); Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List<String> args = getArgListForTable(TABLE_NAME, true, false); SqoopOptions options = new SqoopOptions(newConf()); options.setThrowOnError(true); thrown.expectMessage("--merge-key or --append is required when using --incremental lastmodified and the output directory exists."); Sqoop sqoop = new Sqoop(new ImportTool(), options.getConf(), options); ToolRunner.run(sqoop.getConf(), sqoop, args.toArray(new String[0])); }
@Test public void testTimestampBoundary() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "boundaryTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
@Test public void testIncrementalAppendTimestamp() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "incrementalAppendTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, true, true); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } @Test
@Test public void testEmptyAppendImport() throws Exception { final String TABLE_NAME = "emptyAppend1"; createIdTable(TABLE_NAME, 0); List<String> args = getArgListForTable(TABLE_NAME, true, true); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(TABLE_NAME, 0); }
@Test public void testFullAppendImport() throws Exception { final String TABLE_NAME = "fullAppend1"; createIdTable(TABLE_NAME, 10); List<String> args = getArgListForTable(TABLE_NAME, true, true); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(TABLE_NAME, 10); }
@Test public void testEmptyLastModified() throws Exception { final String TABLE_NAME = "emptyLastModified"; createTimestampTable(TABLE_NAME, 0, null); List<String> args = getArgListForTable(TABLE_NAME, true, false); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(TABLE_NAME, 0); }
@Test public void testEmptyLastModifiedWithNonExistingParentDirectory() throws Exception { final String TABLE_NAME = "emptyLastModifiedNoParent"; final String QUERY = "SELECT id, last_modified FROM \"" + TABLE_NAME + "\" WHERE $CONDITIONS"; final String DIRECTORY = "non-existing/parents/" + TABLE_NAME; createTimestampTable(TABLE_NAME, 0, null); List<String> args = getArgListForQuery(QUERY, DIRECTORY, true, false, false); Configuration conf = newConf(); SqoopOptions options = new SqoopOptions(); options.setConf(conf); runImport(options, args); assertDirOfNumbers(DIRECTORY, 0); }