private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }
@Test public void testHeartbeat() throws Exception { Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); // acquire lock on file1 FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); ArrayList<String> lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 1, lines.size()); // hearbeat upon it lock1.heartbeat("1"); lock1.heartbeat("2"); lock1.heartbeat("3"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 4, lines.size()); lock1.heartbeat("4"); lock1.heartbeat("5"); lock1.heartbeat("6"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 7, lines.size()); lock1.release(); lines = readTextFile(lock1.getLockFile()); Assert.assertNull(lines); Assert.assertFalse(fs.exists(lock1.getLockFile())); }
@Test public void testStaleLockDetection_SingleLock() throws Exception { final int LOCK_EXPIRY_SEC = 1; final int WAIT_MSEC = 1500; Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); try { // acquire lock on file1 Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); Thread.sleep(WAIT_MSEC); // wait for lock to expire HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); // heartbeat, ensure its no longer stale and read back the heartbeat data lock1.heartbeat("1"); expired = FileLock.locateOldestExpiredLock(fs, locksDir, 1); Assert.assertNull(expired); FileLock.LogEntry lastEntry = lock1.getLastLogEntry(); Assert.assertNotNull(lastEntry); Assert.assertEquals("1", lastEntry.fileOffset); // wait and check for expiry again Thread.sleep(WAIT_MSEC); expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); } finally { lock1.release(); fs.delete(file1, false); } }
private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }