private FSDataOutputStream createFileWithRetries(final FileSystem fs, final Path hbckLockFilePath, final FsPermission defaultPerms) throws IOException { IOException exception = null; do { try { return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); } catch (IOException ioe) { LOG.info("Failed to create lock file " + hbckLockFilePath.getName() + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts()); LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), ioe); try { exception = ioe; retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ie) { throw (InterruptedIOException) new InterruptedIOException( "Can't create lock file " + hbckLockFilePath.getName()) .initCause(ie); } } } while (retryCounter.shouldRetry()); throw exception; } }
private void unlockHbck() { if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) { RetryCounter retryCounter = lockFileRetryCounterFactory.create(); do { try { IOUtils.closeQuietly(hbckOutFd); FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true); LOG.info("Finishing hbck"); return; } catch (IOException ioe) { LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts()); LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe); try { retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); LOG.warn("Interrupted while deleting lock file" + HBCK_LOCK_PATH); return; } } } while (retryCounter.shouldRetry()); } }
retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ie) { throw (InterruptedIOException) new InterruptedIOException(
private Pair<Integer, String> execWithRetries(String hostname, ServiceType service, String... cmd) throws IOException { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { return exec(hostname, service, cmd); } catch (IOException e) { retryOrThrow(retryCounter, e, hostname, cmd); } try { retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ex) { // ignore LOG.warn("Sleep Interrupted:" + ex); } } }
retryCounter.sleepUntilNextRetry(); isRetry = true;
retryCounter.sleepUntilNextRetry(); isRetry = true;
/** * getChildren is an idempotent operation. Retry before throwing exception * @return List of children znodes */ public List<String> getChildren(String path, Watcher watcher) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.getChildren")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); List<String> children = checkZk().getChildren(path, watcher); return children; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "getChildren"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getChildren"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * getChildren is an idempotent operation. Retry before throwing exception * @return List of children znodes */ public List<String> getChildren(String path, boolean watch) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.getChildren")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); List<String> children = checkZk().getChildren(path, watch); return children; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "getChildren"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getChildren"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * exists is an idempotent operation. Retry before throwing exception * @return A Stat instance */ public Stat exists(String path, Watcher watcher) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.exists")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); Stat nodeStat = checkZk().exists(path, watcher); return nodeStat; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "exists"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "exists"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * exists is an idempotent operation. Retry before throwing exception * @return A Stat instance */ public Stat exists(String path, boolean watch) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.exists")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); Stat nodeStat = checkZk().exists(path, watch); return nodeStat; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "exists"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "exists"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
retryCounter.sleepUntilNextRetry();
/** * getData is an idempotent operation. Retry before throwing exception * @return Data */ public byte[] getData(String path, boolean watch, Stat stat) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.getData")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); byte[] revData = checkZk().getData(path, watch, stat); return ZKMetadata.removeMetaData(revData); } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "getData"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getData"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * getData is an idempotent operation. Retry before throwing exception * @return Data */ public byte[] getData(String path, Watcher watcher, Stat stat) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.getData")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); byte[] revData = checkZk().getData(path, watcher, stat); return ZKMetadata.removeMetaData(revData); } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "getData"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getData"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
retrier.sleepUntilNextRetry();
@Test public void testBasics() throws InterruptedException { int maxAttempts = 10; RetryCounterFactory factory = new RetryCounterFactory(maxAttempts, 10, 1000); RetryCounter retryCounter = factory.create(); while (retryCounter.shouldRetry()) { LOG.info("Attempt={}, backoffTime={}", retryCounter.getAttemptTimes(), retryCounter.getBackoffTime()); retryCounter.sleepUntilNextRetry(); } assertTrue(retryCounter.getAttemptTimes() == maxAttempts); } }
retryCounter.sleepUntilNextRetry(); isRetry = true;
/** * getAcl is an idempotent operation. Retry before throwing exception * @return list of ACLs */ public List<ACL> getAcl(String path, Stat stat) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.getAcl")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); List<ACL> nodeACL = checkZk().getACL(path, stat); return nodeACL; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "getAcl"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getAcl"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * setAcl is an idempotent operation. Retry before throwing exception * @return list of ACLs */ public Stat setAcl(String path, List<ACL> acls, int version) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.setAcl")) { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); Stat nodeStat = checkZk().setACL(path, acls, version); return nodeStat; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "setAcl"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "setAcl"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
/** * Run multiple operations in a transactional manner. Retry before throwing exception */ public List<OpResult> multi(Iterable<Op> ops) throws KeeperException, InterruptedException { try (TraceScope scope = TraceUtil.createTrace("RecoverableZookeeper.multi")) { RetryCounter retryCounter = retryCounterFactory.create(); Iterable<Op> multiOps = prepareZKMulti(ops); while (true) { try { long startTime = EnvironmentEdgeManager.currentTime(); List<OpResult> opResults = checkZk().multi(multiOps); return opResults; } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: retryOrThrow(retryCounter, e, "multi"); break; case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "multi"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); } } }
.getCompactionStateForRegion(region.getRegionInfo().getRegionName()) && retrier.shouldRetry()) { retrier.sleepUntilNextRetry();