private static void releaseLockAndLog(FileLock fLock, String spoutId) { try { if (fLock != null) { fLock.release(); LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId); } } catch (IOException e) { LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e); } }
LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile()); Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath); String resumeFromOffset = lock.getLastLogEntry().fileOffset; LOG.info("Resuming processing of abandoned file : {}", file);
Assert.assertTrue(fs.exists(lock1a.getLockFile())); Assert.assertEquals(lock1a.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock1a.getLockFile().getName(), file1.getName()); // verify lock filename Assert.assertFalse(fs.exists(lock1a.getLockFile())); Assert.assertTrue(fs.exists(lock1c.getLockFile())); Assert.assertEquals(lock1c.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock1c.getLockFile().getName(), file1.getName()); // verify lock filename Assert.assertTrue(fs.exists(lock2a.getLockFile())); Assert.assertEquals(lock2a.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock2a.getLockFile().getName(), file2.getName()); // verify lock filename Assert.assertFalse(fs.exists(lock2a.getLockFile())); lock1c.release(); Assert.assertFalse(fs.exists(lock1c.getLockFile()));
@Test public void testHeartbeat() throws Exception { Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); // acquire lock on file1 FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); ArrayList<String> lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 1, lines.size()); // hearbeat upon it lock1.heartbeat("1"); lock1.heartbeat("2"); lock1.heartbeat("3"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 4, lines.size()); lock1.heartbeat("4"); lock1.heartbeat("5"); lock1.heartbeat("6"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 7, lines.size()); lock1.release(); lines = readTextFile(lock1.getLockFile()); Assert.assertNull(lines); Assert.assertFalse(fs.exists(lock1.getLockFile())); }
Assert.assertEquals("Expected lock3 file", Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3.getLockFile()); } finally { lock1.release();
@Test public void testStaleLockDetection_SingleLock() throws Exception { final int LOCK_EXPIRY_SEC = 1; final int WAIT_MSEC = 1500; Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); try { // acquire lock on file1 Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); Thread.sleep(WAIT_MSEC); // wait for lock to expire HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); // heartbeat, ensure its no longer stale and read back the heartbeat data lock1.heartbeat("1"); expired = FileLock.locateOldestExpiredLock(fs, locksDir, 1); Assert.assertNull(expired); FileLock.LogEntry lastEntry = lock1.getLastLogEntry(); Assert.assertNotNull(lastEntry); Assert.assertEquals("1", lastEntry.fileOffset); // wait and check for expiry again Thread.sleep(WAIT_MSEC); expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); } finally { lock1.release(); fs.delete(file1, false); } }
Assert.assertTrue(fs.exists(lock.getLockFile())); Assert.assertTrue(fs.exists(lock.getLockFile())); List<String> contents = readTextFile(fs, lock.getLockFile().toString()); Assert.assertFalse(contents.isEmpty()); Assert.assertFalse(fs.exists(lock.getLockFile())); FileReader rdr = getField(spout2, "reader"); Assert.assertNull(rdr);
Assert.assertTrue(fs.exists(lock.getLockFile())); Assert.assertTrue(fs.exists(lock.getLockFile())); List<String> contents = getTextFileContents(fs, lock.getLockFile()); Assert.assertFalse(contents.isEmpty()); Assert.assertFalse(fs.exists(lock.getLockFile())); FileReader rdr = getField(spout2, "reader"); Assert.assertNull(rdr);
private static void releaseLockAndLog(FileLock fLock, String spoutId) { try { if (fLock != null) { fLock.release(); LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId); } } catch (IOException e) { LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e); } }
LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile()); Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath); String resumeFromOffset = lock.getLastLogEntry().fileOffset; LOG.info("Resuming processing of abandoned file : {}", file);