public static boolean isInitialized(FluoConfiguration config) { try (CuratorFramework rootCurator = CuratorUtil.newRootFluoCurator(config)) { rootCurator.start(); String appRootDir = ZookeeperUtil.parseRoot(config.getAppZookeepers()); return rootCurator.checkExists().forPath(appRootDir) != null; } catch (Exception e) { throw new IllegalStateException(e); } }
public static boolean isInitialized(FluoConfiguration config) { try (CuratorFramework rootCurator = CuratorUtil.newRootFluoCurator(config)) { rootCurator.start(); String appRootDir = ZookeeperUtil.parseRoot(config.getAppZookeepers()); return rootCurator.checkExists().forPath(appRootDir) != null; } catch (Exception e) { throw new IllegalStateException(e); } }
private ZooKeeper getZookeeper() throws IOException { ZooKeeper zk = new ZooKeeper(config.getAppZookeepers(), 30000, null); long start = System.currentTimeMillis(); while (!zk.getState().isConnected() && System.currentTimeMillis() - start < 30000) { Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS); } return zk; }
public static void verifyAppInitialized(FluoConfiguration config) { if (!FluoAdminImpl.isInitialized(config)) { System.out.println("A Fluo '" + config.getApplicationName() + "' application has not " + "been initialized yet in Zookeeper at " + config.getAppZookeepers()); System.exit(-1); } }
public FluoAdminImpl(FluoConfiguration config) { this.config = config; appRootDir = ZookeeperUtil.parseRoot(config.getAppZookeepers()); rootCurator = CuratorUtil.newRootFluoCurator(config); rootCurator.start(); }
public FluoAdminImpl(FluoConfiguration config) { this.config = config; appRootDir = ZookeeperUtil.parseRoot(config.getAppZookeepers()); rootCurator = CuratorUtil.newRootFluoCurator(config); rootCurator.start(); }
private void waitForGcTime(long expectedTime) throws Exception { env.getSharedResources().getTimestampTracker().updateZkNode(); long oldestTs = ZookeeperUtil.getGcTimestamp(config.getAppZookeepers()); while (oldestTs < expectedTime) { Thread.sleep(500); oldestTs = ZookeeperUtil.getGcTimestamp(config.getAppZookeepers()); } }
@Test public void testGetOldestTimestamp() throws Exception { // we are expecting an error in this test final Level curLevel = Logger.getLogger(ZookeeperUtil.class).getLevel(); Logger.getLogger(ZookeeperUtil.class).setLevel(Level.FATAL); // verify that oracle initial current ts Assert.assertEquals(0, ZookeeperUtil.getGcTimestamp(config.getAppZookeepers())); // delete the oracle current timestamp path env.getSharedResources().getCurator().delete().forPath(ZookeeperPath.ORACLE_GC_TIMESTAMP); // verify that oldest possible is returned Assert.assertEquals(ZookeeperUtil.OLDEST_POSSIBLE, ZookeeperUtil.getGcTimestamp(config.getAppZookeepers())); // set level back Logger.getLogger(ZookeeperUtil.class).setLevel(curLevel); }
/** * Creates a curator built using Application's zookeeper connection string. Root path will start * at Fluo application chroot. */ public static CuratorFramework newAppCurator(FluoConfiguration config) { return newCurator(config.getAppZookeepers(), config.getZookeeperTimeout(), config.getZookeeperSecret()); }
/** * Creates a curator built using Application's zookeeper connection string. Root path will start * at Fluo application chroot. */ public static CuratorFramework newAppCurator(FluoConfiguration config) { return newCurator(config.getAppZookeepers(), config.getZookeeperTimeout(), config.getZookeeperSecret()); }
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start(); twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start(); twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
private Map<String, String> initializeApplicationTableProps() { Map<String, String> ntcProps = new HashMap<>(); ntcProps.put(AccumuloProps.TABLE_GROUP_PREFIX + ColumnConstants.NOTIFY_LOCALITY_GROUP_NAME, encodeColumnFamily(ColumnConstants.NOTIFY_CF)); ntcProps.put(AccumuloProps.TABLE_GROUPS_ENABLED, ColumnConstants.NOTIFY_LOCALITY_GROUP_NAME); IteratorSetting gcIter = new IteratorSetting(10, ColumnConstants.GC_CF.toString(), GarbageCollectionIterator.class); GarbageCollectionIterator.setZookeepers(gcIter, config.getAppZookeepers()); // the order relative to gc iter should not matter IteratorSetting ntfyIter = new IteratorSetting(11, ColumnConstants.NOTIFY_CF.toString(), NotificationIterator.class); for (IteratorSetting setting : new IteratorSetting[] {gcIter, ntfyIter}) { for (IteratorScope scope : EnumSet.of(IteratorUtil.IteratorScope.majc, IteratorUtil.IteratorScope.minc)) { String root = String.format("%s%s.%s", AccumuloProps.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), setting.getName()); for (Entry<String, String> prop : setting.getOptions().entrySet()) { ntcProps.put(root + ".opt." + prop.getKey(), prop.getValue()); } ntcProps.put(root, setting.getPriority() + "," + setting.getIteratorClass()); } } return ntcProps; }
private void checkIfInitialized(FluoConfiguration config) { try (FluoAdminImpl admin = new FluoAdminImpl(config)) { if (!admin.zookeeperInitialized()) { throw new FluoException("A Fluo '" + config.getApplicationName() + "' application has not " + "been initialized yet in Zookeeper at " + config.getAppZookeepers()); } } }
private void checkIfInitialized(FluoConfiguration config) { try (FluoAdminImpl admin = new FluoAdminImpl(config)) { if (!admin.zookeeperInitialized()) { throw new FluoException("A Fluo '" + config.getApplicationName() + "' application has not " + "been initialized yet in Zookeeper at " + config.getAppZookeepers()); } } }
private void configureIterators(NewTableConfiguration ntc) { IteratorSetting gcIter = new IteratorSetting(10, ColumnConstants.GC_CF.toString(), GarbageCollectionIterator.class); GarbageCollectionIterator.setZookeepers(gcIter, config.getAppZookeepers()); // the order relative to gc iter should not matter IteratorSetting ntfyIter = new IteratorSetting(11, ColumnConstants.NOTIFY_CF.toString(), NotificationIterator.class); EnumSet<IteratorScope> scopes = EnumSet.of(IteratorUtil.IteratorScope.majc, IteratorUtil.IteratorScope.minc); ntc.attachIterator(gcIter, scopes); ntc.attachIterator(ntfyIter, scopes); }
@Test public void testInitializeLongChroot() throws Exception { // stop oracle to avoid spurious exceptions when initializing oserver.stop(); String zk = config.getAppZookeepers(); String longPath = "/very/long/path"; config.setInstanceZookeepers(zk + longPath); InitializationOptions opts = new InitializationOptions(); opts.setClearZookeeper(true).setClearTable(true); try (FluoAdmin admin = new FluoAdminImpl(config)) { admin.initialize(opts); } try (CuratorFramework curator = CuratorUtil.newRootFluoCurator(config)) { curator.start(); Assert.assertNotNull(curator.checkExists().forPath(ZookeeperUtil.parseRoot(zk + longPath))); } String longPath2 = "/very/long/path2"; config.setInstanceZookeepers(zk + longPath2); try (FluoAdmin admin = new FluoAdminImpl(config)) { admin.initialize(opts); } try (CuratorFramework curator = CuratorUtil.newRootFluoCurator(config)) { curator.start(); Assert.assertNotNull(curator.checkExists().forPath(ZookeeperUtil.parseRoot(zk + longPath2))); Assert.assertNotNull(curator.checkExists().forPath(ZookeeperUtil.parseRoot(zk + longPath))); } }
public void validate() { // keep in alphabetical order getAccumuloInstance(); getAccumuloPassword(); getAccumuloTable(); getAccumuloUser(); getAccumuloZookeepers(); getApplicationName(); getAppZookeepers(); getConnectionRetryTimeout(); getLoaderQueueSize(); getLoaderThreads(); getObserverSpecifications(); getTransactionRollbackTime(); getWorkerThreads(); getZookeeperTimeout(); }
public void validate() { // keep in alphabetical order getAccumuloInstance(); getAccumuloPassword(); getAccumuloTable(); getAccumuloUser(); getAccumuloZookeepers(); getApplicationName(); getAppZookeepers(); getConnectionRetryTimeout(); getLoaderQueueSize(); getLoaderThreads(); getObserverSpecifications(); getTransactionRollbackTime(); getWorkerThreads(); getZookeeperTimeout(); }
public void testVerifyAfterGC() throws Exception { final TestTransaction tx1 = new TestTransaction(env); BankUtil.setBalance(tx1, "bob", 10); BankUtil.setBalance(tx1, "joe", 20); BankUtil.setBalance(tx1, "jill", 60); tx1.done(); BankUtil.transfer(env, "joe", "jill", 1); BankUtil.transfer(env, "joe", "bob", 1); BankUtil.transfer(env, "bob", "joe", 2); BankUtil.transfer(env, "jill", "joe", 2); final TestTransaction tx2 = new TestTransaction(env); waitForGcTime(tx2.getStartTimestamp()); long oldestTs = ZookeeperUtil.getGcTimestamp(config.getAppZookeepers()); Assert.assertEquals(tx2.getStartTs(), oldestTs); // Force a garbage collection aClient.tableOperations().flush(table, null, null, true); verify(oldestTs); final TestTransaction tx3 = new TestTransaction(env); Assert.assertEquals(9, BankUtil.getBalance(tx3, "bob")); Assert.assertEquals(22, BankUtil.getBalance(tx3, "joe")); Assert.assertEquals(59, BankUtil.getBalance(tx3, "jill")); tx3.done(); tx2.done(); }