FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2"); FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3"); Assert.assertNotNull(lock1); Assert.assertNotNull(lock2); HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNull(expired); lock1.heartbeat("1"); lock2.heartbeat("1"); FileLock lock3b = FileLock.acquireOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC, "spout1"); Assert.assertNotNull(lock3b); Assert.assertEquals("Expected lock3 file", Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3.getLockFile()); } finally { lock1.release(); lock2.release(); lock3.release(); fs.delete(file1, false); fs.delete(file2, false);
@Test public void testStaleLockDetection_SingleLock() throws Exception { final int LOCK_EXPIRY_SEC = 1; final int WAIT_MSEC = 1500; Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); try { // acquire lock on file1 Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); Thread.sleep(WAIT_MSEC); // wait for lock to expire HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); // heartbeat, ensure its no longer stale and read back the heartbeat data lock1.heartbeat("1"); expired = FileLock.locateOldestExpiredLock(fs, locksDir, 1); Assert.assertNull(expired); FileLock.LogEntry lastEntry = lock1.getLastLogEntry(); Assert.assertNotNull(lastEntry); Assert.assertEquals("1", lastEntry.fileOffset); // wait and check for expiry again Thread.sleep(WAIT_MSEC); expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); } finally { lock1.release(); fs.delete(file1, false); } }
/** returns lock on file or null if file is already locked. throws if unexpected problem */ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId) throws IOException { Path lockFile = new Path(lockDirPath, fileToLock.getName()); try { FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile); if (ostream != null) { LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId); return new FileLock(fs, lockFile, ostream, spoutId); } else { LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId); return null; } } catch (IOException e) { LOG.error("Error when acquiring lock on file " + fileToLock + " Spout = " + spoutId, e); throw e; } }
private static void releaseLockAndLog(FileLock fLock, String spoutId) { try { if (fLock != null) { fLock.release(); LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId); } } catch (IOException e) { LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e); } }
continue; LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan); if (lastEntry != null) { FileLock lock = FileLock.takeOwnership(fs, file, lastEntry, spoutId); if (lock != null) { return lock;
LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile()); Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath); String resumeFromOffset = lock.getLastLogEntry().fileOffset; LOG.info("Resuming processing of abandoned file : {}", file); return createFileReader(file, resumeFromOffset); continue; lock = FileLock.tryLock(hdfs, file, lockDirPath, spoutId); if (lock == null) { LOG.debug("Unable to get FileLock for {}, so skipping it.", file);
@Test public void testHeartbeat() throws Exception { Path file1 = new Path(filesDir + Path.SEPARATOR + "file1"); fs.create(file1).close(); // acquire lock on file1 FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNotNull(lock1); Assert.assertTrue(fs.exists(lock1.getLockFile())); ArrayList<String> lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 1, lines.size()); // hearbeat upon it lock1.heartbeat("1"); lock1.heartbeat("2"); lock1.heartbeat("3"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 4, lines.size()); lock1.heartbeat("4"); lock1.heartbeat("5"); lock1.heartbeat("6"); lines = readTextFile(lock1.getLockFile()); Assert.assertEquals("heartbeats appear to be missing", 7, lines.size()); lock1.release(); lines = readTextFile(lock1.getLockFile()); Assert.assertNull(lines); Assert.assertFalse(fs.exists(lock1.getLockFile())); }
FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1"); FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2"); FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3"); Assert.assertNotNull(lock1); Assert.assertNotNull(lock2); HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNull(expired); lock1.heartbeat("1"); lock2.heartbeat("1"); expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); Assert.assertNotNull(expired); Assert.assertEquals("spout3", expired.getValue().componentID); } finally { lock1.release(); lock2.release(); lock3.release(); fs.delete(file1, false); fs.delete(file2, false);
return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId); lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec); lastExpiredLockTime = System.currentTimeMillis(); return null; FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey()); if (lastEntry.equals(lastExpiredLock.getValue())) { FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId); lastExpiredLock = null; return result;
FileLock lock1a = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNotNull(lock1a); Assert.assertTrue(fs.exists(lock1a.getLockFile())); Assert.assertEquals(lock1a.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock1a.getLockFile().getName(), file1.getName()); // verify lock filename FileLock lock1b = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNull(lock1b); lock1a.release(); Assert.assertFalse(fs.exists(lock1a.getLockFile())); FileLock lock1c = FileLock.tryLock(fs, file1, locksDir, "spout1"); Assert.assertNotNull(lock1c); Assert.assertTrue(fs.exists(lock1c.getLockFile())); Assert.assertEquals(lock1c.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock1c.getLockFile().getName(), file1.getName()); // verify lock filename FileLock lock2a = FileLock.tryLock(fs, file2, locksDir, "spout1"); Assert.assertNotNull(lock2a); Assert.assertTrue(fs.exists(lock2a.getLockFile())); Assert.assertEquals(lock2a.getLockFile().getParent(), locksDir); // verify lock file location Assert.assertEquals(lock2a.getLockFile().getName(), file2.getName()); // verify lock filename lock2a.release(); Assert.assertFalse(fs.exists(lock2a.getLockFile())); lock1c.release(); Assert.assertFalse(fs.exists(lock1c.getLockFile()));
@Override public void run() { Thread.currentThread().setName("FileLockingThread-" + thdNum); FileLock lock = null; try { do { System.err.println("Trying lock - " + getName()); lock = FileLock.tryLock(fs, this.fileToLock, this.locksDir, spoutId); System.err.println("Acquired lock - " + getName()); if (lock == null) { System.out.println("Retrying lock - " + getName()); } } while (lock == null && !Thread.currentThread().isInterrupted()); cleanExit = true; } catch (Exception e) { e.printStackTrace(); } finally { try { if (lock != null) { lock.release(); System.err.println("Released lock - " + getName()); } } catch (IOException e) { e.printStackTrace(System.err); } } System.err.println("Thread exiting - " + getName()); } // run()
private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }
Assert.assertTrue(fs.exists(lock.getLockFile())); Assert.assertTrue(fs.exists(lock.getLockFile())); List<String> contents = readTextFile(fs, lock.getLockFile().toString()); Assert.assertFalse(contents.isEmpty()); Assert.assertFalse(fs.exists(lock.getLockFile())); FileReader rdr = getField(spout2, "reader"); Assert.assertNull(rdr);
/** * Finds oldest expired lock file (using modification timestamp), then takes * ownership of the lock file * Impt: Assumes access to lockFilesDir has been externally synchronized such that * only one thread accessing the same thread * @param fs * @param lockFilesDir * @param locktimeoutSec * @return a Pair<lock file path, last entry in lock file> .. if expired lock file found * @throws IOException */ public static HdfsUtils.Pair<Path, LogEntry> locateOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec) throws IOException { // list files long now = System.currentTimeMillis(); long olderThan = now - (locktimeoutSec * 1000); Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan); // locate oldest expired lock file (if any) and take ownership for (Path file : listing) { if (file.getName().equalsIgnoreCase(DirLock.DIR_LOCK_FILE)) { continue; } LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan); if (lastEntry != null) { return new HdfsUtils.Pair<>(file, lastEntry); } } LOG.debug("No abandoned files found"); return null; }
LogEntry lastEntry = getLastEntry(fs, lockFile); if (lastEntry == null) { LOG.warn("Empty lock file found. Deleting it. {}", lockFile);
LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile()); Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath); String resumeFromOffset = lock.getLastLogEntry().fileOffset; LOG.info("Resuming processing of abandoned file : {}", file); return createFileReader(file, resumeFromOffset); continue; lock = FileLock.tryLock(hdfs, file, lockDirPath, spoutId); if (lock == null) { LOG.debug("Unable to get FileLock for {}, so skipping it.", file);
return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId); lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec); lastExpiredLockTime = System.currentTimeMillis(); return null; FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey()); if (lastEntry.equals(lastExpiredLock.getValue())) { FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId); lastExpiredLock = null; return result;
private static void releaseLockAndLog(FileLock fLock, String spoutId) { try { if (fLock != null) { fLock.release(); LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId); } } catch (IOException e) { LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e); } }
private void commitProgress(FileOffset position) { if (position == null) { return; } if (lock != null && canCommitNow()) { try { String pos = position.toString(); lock.heartbeat(pos); LOG.debug("{} Committed progress. {}", spoutId, pos); acksSinceLastCommit = 0; commitTimeElapsed.set(false); setupCommitElapseTimer(); } catch (IOException e) { LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e); } } }
continue; LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan); if(lastEntry!=null) { FileLock lock = FileLock.takeOwnership(fs, file, lastEntry, spoutId); if(lock!=null) { return lock;