public boolean accumuloTableExists() { if (!config.hasRequiredAdminProps()) { throw new IllegalArgumentException("Admin configuration is missing required properties"); } Connector conn = AccumuloUtil.getConnector(config); return conn.tableOperations().exists(config.getAccumuloTable()); } }
public boolean accumuloTableExists() { if (!config.hasRequiredAdminProps()) { throw new IllegalArgumentException("Admin configuration is missing required properties"); } AccumuloClient client = AccumuloUtil.getClient(config); return client.tableOperations().exists(config.getAccumuloTable()); } }
boolean tableExists = conn.tableOperations().exists(config.getAccumuloTable()); logger.info("The Accumulo table '{}' will be dropped", config.getAccumuloTable()); try { conn.tableOperations().delete(config.getAccumuloTable()); } catch (Exception e) { throw new RuntimeException(e);
public static void compactTransient(FluoConfiguration fluoConfig, RowRange tRange) throws Exception { Connector conn = getConnector(fluoConfig); conn.tableOperations().compact(fluoConfig.getAccumuloTable(), new Text(tRange.getStart().toArray()), new Text(tRange.getEnd().toArray()), true, true); } }
/** * Compact all transient regions that were registered using {@link TransientRegistry} */ public static void compactTransient(FluoConfiguration fluoConfig) throws Exception { Connector conn = getConnector(fluoConfig); try (FluoClient client = FluoFactory.newClient(fluoConfig)) { SimpleConfiguration appConfig = client.getAppConfiguration(); TransientRegistry transientRegistry = new TransientRegistry(appConfig); List<RowRange> ranges = transientRegistry.getTransientRanges(); for (RowRange r : ranges) { long t1 = System.currentTimeMillis(); conn.tableOperations().compact(fluoConfig.getAccumuloTable(), new Text(r.getStart().toArray()), new Text(r.getEnd().toArray()), true, true); long t2 = System.currentTimeMillis(); logger.info("Compacted {} in {}ms", r, (t2 - t1)); } } }
boolean tableExists = conn.tableOperations().exists(config.getAccumuloTable()); logger.info("The Accumulo table '{}' will be dropped", config.getAccumuloTable()); try { conn.tableOperations().delete(config.getAccumuloTable()); } catch (Exception e) { throw new RuntimeException(e);
public static void scanAccumulo(ScanOpts options, FluoConfiguration sConfig) { Connector conn = AccumuloUtil.getConnector(sConfig); Span span = null; Collection<Column> columns = null; try { span = getSpan(options); columns = getColumns(options); } catch (IllegalArgumentException e) { System.err.println(e.getMessage()); System.exit(-1); } try { Scanner scanner = conn.createScanner(sConfig.getAccumuloTable(), Authorizations.EMPTY); scanner.setRange(SpanUtil.toRange(span)); for (Column col : columns) { if (col.isQualifierSet()) { scanner.fetchColumn(ByteUtil.toText(col.getFamily()), ByteUtil.toText(col.getQualifier())); } else { scanner.fetchColumnFamily(ByteUtil.toText(col.getFamily())); } } for (String entry : Iterables.transform(scanner, FluoFormatter::toString)) { System.out.println(entry); } } catch (Exception e) { System.err.println("Scan failed - " + e.getMessage()); System.exit(-1); } }
private void initializeApplicationInZooKeeper(AccumuloClient client) throws Exception { final String accumuloInstanceName = client.info().getInstanceName(); final String accumuloInstanceID = client.getInstanceID(); final String fluoApplicationID = UUID.randomUUID().toString(); // Create node specified by chroot suffix of Zookeeper connection string (if it doesn't exist) CuratorUtil.putData(rootCurator, appRootDir, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); // Retrieve Fluo curator now that chroot has been created CuratorFramework curator = getAppCurator(); // Initialize Zookeeper & Accumulo for this Fluo instance // TODO set Fluo data version CuratorUtil.putData(curator, ZookeeperPath.CONFIG, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_TABLE, config.getAccumuloTable().getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_NAME, accumuloInstanceName.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_ID, accumuloInstanceID.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_FLUO_APPLICATION_ID, fluoApplicationID.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_SERVER, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_MAX_TIMESTAMP, new byte[] {'2'}, CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_GC_TIMESTAMP, new byte[] {'0'}, CuratorUtil.NodeExistsPolicy.FAIL); }
boolean tableExists = conn.tableOperations().exists(config.getAccumuloTable()); if (tableExists && !opts.getClearTable()) { throw new TableExistsException("Accumulo table already exists " + config.getAccumuloTable()); config.getAccumuloTable()); try { conn.tableOperations().delete(config.getAccumuloTable()); } catch (Exception e) { throw new RuntimeException(e); conn.tableOperations().create(config.getAccumuloTable(), ntc);
private void initializeApplicationInZooKeeper(Connector conn) throws Exception { final String accumuloInstanceName = conn.getInstance().getInstanceName(); final String accumuloInstanceID = conn.getInstance().getInstanceID(); final String fluoApplicationID = UUID.randomUUID().toString(); // Create node specified by chroot suffix of Zookeeper connection string (if it doesn't exist) CuratorUtil.putData(rootCurator, appRootDir, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); // Retrieve Fluo curator now that chroot has been created CuratorFramework curator = getAppCurator(); // Initialize Zookeeper & Accumulo for this Fluo instance // TODO set Fluo data version CuratorUtil.putData(curator, ZookeeperPath.CONFIG, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_TABLE, config.getAccumuloTable().getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_NAME, accumuloInstanceName.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_ACCUMULO_INSTANCE_ID, accumuloInstanceID.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.CONFIG_FLUO_APPLICATION_ID, fluoApplicationID.getBytes(StandardCharsets.UTF_8), CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_SERVER, new byte[0], CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_MAX_TIMESTAMP, new byte[] {'2'}, CuratorUtil.NodeExistsPolicy.FAIL); CuratorUtil.putData(curator, ZookeeperPath.ORACLE_GC_TIMESTAMP, new byte[] {'0'}, CuratorUtil.NodeExistsPolicy.FAIL); }
String table = fluoConfig.getAccumuloTable(); conn.tableOperations().addSplits(table, splits);
client.tableOperations().getLocalityGroups(config.getAccumuloTable()); Assert.assertEquals("Unexpected locality group count.", 1, localityGroups.size()); Entry<String, Set<Text>> localityGroup = localityGroups.entrySet().iterator().next();
private void startMiniAccumulo() { try { // start mini accumulo cluster MiniAccumuloConfig cfg = new MiniAccumuloConfig(new File(config.getMiniDataDir()), PASSWORD); cluster = new MiniAccumuloCluster(cfg); cluster.start(); log.debug("Started MiniAccumulo(accumulo=" + cluster.getInstanceName() + " zk=" + cluster.getZooKeepers() + ")"); // configuration that must overridden config.setAccumuloInstance(cluster.getInstanceName()); config.setAccumuloUser(USER); config.setAccumuloPassword(PASSWORD); config.setAccumuloZookeepers(cluster.getZooKeepers()); config.setInstanceZookeepers(cluster.getZooKeepers() + "/fluo"); // configuration that only needs to be set if not by user if ((config.containsKey(FluoConfiguration.ACCUMULO_TABLE_PROP) == false) || config.getAccumuloTable().trim().isEmpty()) { config.setAccumuloTable("fluo"); } InitializationOptions opts = new InitializationOptions(); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { admin.initialize(opts); } File miniProps = new File(clientPropsPath(config)); config.getClientConfiguration().save(miniProps); log.debug("Wrote MiniFluo client properties to {}", miniProps.getAbsolutePath()); } catch (Exception e) { throw new RuntimeException(e); } }
private void startMiniAccumulo() { try { // start mini accumulo cluster MiniAccumuloConfig cfg = new MiniAccumuloConfig(new File(config.getMiniDataDir()), PASSWORD); cluster = new MiniAccumuloCluster(cfg); cluster.start(); log.debug("Started MiniAccumulo(accumulo=" + cluster.getInstanceName() + " zk=" + cluster.getZooKeepers() + ")"); // configuration that must overridden config.setAccumuloInstance(cluster.getInstanceName()); config.setAccumuloUser(USER); config.setAccumuloPassword(PASSWORD); config.setAccumuloZookeepers(cluster.getZooKeepers()); config.setInstanceZookeepers(cluster.getZooKeepers() + "/fluo"); // configuration that only needs to be set if not by user if ((config.containsKey(FluoConfiguration.ACCUMULO_TABLE_PROP) == false) || config.getAccumuloTable().trim().isEmpty()) { config.setAccumuloTable("fluo"); } InitializationOptions opts = new InitializationOptions(); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { admin.initialize(opts); } File miniProps = new File(clientPropsPath(config)); config.getClientConfiguration().save(miniProps); log.debug("Wrote MiniFluo client properties to {}", miniProps.getAbsolutePath()); } catch (Exception e) { throw new RuntimeException(e); } }
public static void scanAccumulo(ScanOpts options, FluoConfiguration sConfig, PrintStream out) { AccumuloClient client = AccumuloUtil.getClient(sConfig); Span span = getSpan(options); Collection<Column> columns = getColumns(options); try { Scanner scanner = client.createScanner(sConfig.getAccumuloTable(), Authorizations.EMPTY); scanner.setRange(SpanUtil.toRange(span)); for (Column col : columns) { if (col.isQualifierSet()) { scanner.fetchColumn(ByteUtil.toText(col.getFamily()), ByteUtil.toText(col.getQualifier())); } else { scanner.fetchColumnFamily(ByteUtil.toText(col.getFamily())); } } for (String entry : Iterables.transform(scanner, FluoFormatter::toString)) { out.println(entry); } out.flush(); } catch (Exception e) { throw new RuntimeException(e); } }
public static void scanNotifications(ScanOpts options, FluoConfiguration sConfig, PrintStream out) throws IOException { AccumuloClient client = AccumuloUtil.getClient(sConfig); Span span = getSpan(options); Collection<Column> columns = getColumns(options); Scanner scanner = null; try { scanner = client.createScanner(sConfig.getAccumuloTable(), Authorizations.EMPTY); scanner.setRange(SpanUtil.toRange(span)); NotificationScanner ntfyScanner = new NotificationScanner(scanner, columns); scan(options, out, ntfyScanner); } catch (TableNotFoundException e) { throw new RuntimeException(e); } finally { if (scanner != null) { scanner.close(); } } }
public void validate() { // keep in alphabetical order getAccumuloInstance(); getAccumuloPassword(); getAccumuloTable(); getAccumuloUser(); getAccumuloZookeepers(); getApplicationName(); getAppZookeepers(); getConnectionRetryTimeout(); getLoaderQueueSize(); getLoaderThreads(); getObserverSpecifications(); getTransactionRollbackTime(); getWorkerThreads(); getZookeeperTimeout(); }
public void validate() { // keep in alphabetical order getAccumuloInstance(); getAccumuloPassword(); getAccumuloTable(); getAccumuloUser(); getAccumuloZookeepers(); getApplicationName(); getAppZookeepers(); getConnectionRetryTimeout(); getLoaderQueueSize(); getLoaderThreads(); getObserverSpecifications(); getTransactionRollbackTime(); getWorkerThreads(); getZookeeperTimeout(); }
@Test public void testBasic() { try (FluoClient client = FluoFactory.newClient(config)) { client.newSnapshot(); } FluoConfiguration fluoConfig = new FluoConfiguration(); fluoConfig.setApplicationName(config.getApplicationName()); fluoConfig.setInstanceZookeepers(config.getInstanceZookeepers()); try (FluoClient client = FluoFactory.newClient(fluoConfig)) { client.newSnapshot(); } try (FluoClientImpl client = new FluoClientImpl(fluoConfig)) { client.newSnapshot(); FluoConfiguration sharedConfig = client.getSharedConfiguration(); Assert.assertEquals(config.getAccumuloTable(), sharedConfig.getAccumuloTable()); Assert.assertEquals(config.getAccumuloInstance(), sharedConfig.getAccumuloInstance()); Assert.assertEquals(config.getAccumuloUser(), sharedConfig.getAccumuloUser()); Assert.assertEquals(config.getZookeeperTimeout(), sharedConfig.getZookeeperTimeout()); Assert.assertEquals(config.getTransactionRollbackTime(), sharedConfig.getTransactionRollbackTime()); } }
assertTrue(aClient.tableOperations().exists(config.getAccumuloTable()));