@Test public void testTimestampBoundary() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "boundaryTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
@Test public void testIncrementalAppendTimestamp() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "incrementalAppendTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, true, true); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } @Test