public static AccumuloGraphConfiguration getAccumuloGraphConfiguration(Configuration conf) { return new AccumuloGraphConfiguration(conf, conf.get("MRMigrationBase.config.prefix")); }
public static AccumuloGraph create(AccumuloGraphConfiguration config) { if (config == null) { throw new IllegalArgumentException("config cannot be null"); } Connector connector = config.createConnector(); if (config.isHistoryInSeparateTable()) { ensureTableExists(connector, getVerticesTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getEdgesTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getHistoryVerticesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getHistoryEdgesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureRowDeletingIteratorIsAttached(connector, getHistoryVerticesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getHistoryEdgesTableName(config.getTableNamePrefix())); } else { ensureTableExists(connector, getVerticesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getEdgesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); } ensureTableExists(connector, getExtendedDataTableName(config.getTableNamePrefix()), config.getExtendedDataMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getDataTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getMetadataTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureRowDeletingIteratorIsAttached(connector, getVerticesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getEdgesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getDataTableName(config.getTableNamePrefix())); AccumuloGraph graph = new AccumuloGraph(config, connector); graph.setup(); return graph; }
public static VertexiumSerializer getVertexiumSerializer(Configuration conf) { return getAccumuloGraphConfiguration(conf).createSerializer(); }
public Connector createConnector() { try { LOGGER.info("Connecting to accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()); ZooKeeperInstance instance = new ZooKeeperInstance(getClientConfiguration()); return instance.getConnector(this.getAccumuloUsername(), this.getAuthenticationToken()); } catch (Exception ex) { throw new VertexiumException( String.format("Could not connect to Accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()), ex ); } }
@SuppressWarnings("unchecked") public ClientConfiguration getClientConfiguration() { ClientConfiguration config = ClientConfiguration.create() .withInstance(this.getAccumuloInstanceName()) .withZkHosts(this.getZookeeperServers()); for (Map.Entry<String, String> entry : getClientConfigurationProperties().entrySet()) { config.setProperty(entry.getKey(), entry.getValue()); } return config; }
protected AccumuloGraph(AccumuloGraphConfiguration config, Connector connector) { super(config); this.connector = connector; this.vertexiumSerializer = config.createSerializer(this); this.nameSubstitutionStrategy = AccumuloNameSubstitutionStrategy.create(config.createSubstitutionStrategy(this)); this.streamingPropertyValueStorageStrategy = config.createStreamingPropertyValueStorageStrategy(this); this.elementMutationBuilder = new ElementMutationBuilder(streamingPropertyValueStorageStrategy, vertexiumSerializer) { @Override curatorFramework = CuratorFrameworkFactory.newClient(config.getZookeeperServers(), retryPolicy); curatorFramework.start(); String zkPath = config.getZookeeperMetadataSyncPath(); this.graphMetadataStore = new AccumuloGraphMetadataStore(curatorFramework, zkPath); this.verticesTableName = getVerticesTableName(getConfiguration().getTableNamePrefix()); this.edgesTableName = getEdgesTableName(getConfiguration().getTableNamePrefix()); this.extendedDataTableName = getExtendedDataTableName(getConfiguration().getTableNamePrefix()); this.dataTableName = getDataTableName(getConfiguration().getTableNamePrefix()); this.metadataTableName = getMetadataTableName(getConfiguration().getTableNamePrefix()); this.numberOfQueryThreads = getConfiguration().getNumberOfQueryThreads(); this.historyInSeparateTable = getConfiguration().isHistoryInSeparateTable(); this.historyVerticesTableName = getHistoryVerticesTableName(getConfiguration().getTableNamePrefix()); this.historyEdgesTableName = getHistoryEdgesTableName(getConfiguration().getTableNamePrefix()); } else { this.historyVerticesTableName = null; BatchWriterConfig writerConfig = getConfiguration().createBatchWriterConfig(); this.batchWriter = connector.createMultiTableBatchWriter(writerConfig);
public static void configure(Job job) { try { AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(job.getConfiguration(), "graph."); AccumuloGraph graph = AccumuloGraph.create(accumuloGraphConfiguration); String principal = accumuloGraphConfiguration.getAccumuloUsername(); AuthenticationToken token = accumuloGraphConfiguration.getAuthenticationToken(); String instanceName = accumuloGraphConfiguration.getAccumuloInstanceName(); String zooKeepers = accumuloGraphConfiguration.getZookeeperServers(); String[] authorizations = job.getConfiguration().get("titan.hadoop.input.authorizations", "").split(","); if (authorizations.length == 1 && authorizations[0].trim().length() == 0) { authorizations = new String[0]; } String tableName = graph.getVerticesTableName(); AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = new ClientConfiguration() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloRowInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(VertexiumMRUtils.CONFIG_AUTHORIZATIONS, authorizations); } catch (Exception ex) { throw new VertexiumException("Could not configure", ex); } }
super.setup(context); String configPrefix = context.getConfiguration().get(GRAPH_CONFIG_PREFIX, ""); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(context.getConfiguration(), configPrefix); String tableNamePrefix = accumuloGraphConfiguration.getTableNamePrefix(); final Text edgesTableName = new Text(AccumuloGraph.getEdgesTableName(tableNamePrefix)); final Text dataTableName = new Text(AccumuloGraph.getDataTableName(tableNamePrefix)); VertexiumSerializer vertexiumSerializer = accumuloGraphConfiguration.createSerializer(this.graph); nameSubstitutionStrategy = accumuloGraphConfiguration.createSubstitutionStrategy(this.graph); StreamingPropertyValueStorageStrategy streamingPropertyValueStorageStrategy = accumuloGraphConfiguration.createStreamingPropertyValueStorageStrategy(this.graph); this.elementMutationBuilder = new ElementMutationBuilder(streamingPropertyValueStorageStrategy, vertexiumSerializer) { @Override
@Test public void testBatchWriterConfigUsesDefaultWithNoParameters() { Map configMap = Maps.newHashMap(); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(configMap); BatchWriterConfig batchWriterConfig = accumuloGraphConfiguration.createBatchWriterConfig(); assertThat(batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_LATENCY)); assertThat(batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_TIMEOUT)); assertThat(batchWriterConfig.getMaxMemory(), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_MEMORY)); assertThat(batchWriterConfig.getMaxWriteThreads(), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_WRITE_THREADS)); }
public Connector createConnector() throws AccumuloSecurityException, AccumuloException { return new AccumuloGraphConfiguration(createConfig()).createConnector(); }
BatchWriter writer = graph.getConnector().createBatchWriter( graph.getDataTableName(), graph.getConfiguration().createBatchWriterConfig() ); String lastRowIdPrefix = null;
@Override public String toString() { return getClass().getSimpleName().toLowerCase() + ":" + getGraph().getConfiguration().getTableNamePrefix(); }
protected AccumuloGraph(AccumuloGraphConfiguration config, Connector connector) { super(config); this.connector = connector; this.vertexiumSerializer = config.createSerializer(this); this.nameSubstitutionStrategy = AccumuloNameSubstitutionStrategy.create(config.createSubstitutionStrategy(this)); this.streamingPropertyValueStorageStrategy = config.createStreamingPropertyValueStorageStrategy(this); this.elementMutationBuilder = new ElementMutationBuilder(streamingPropertyValueStorageStrategy, vertexiumSerializer) { @Override curatorFramework = CuratorFrameworkFactory.newClient(config.getZookeeperServers(), retryPolicy); curatorFramework.start(); String zkPath = config.getZookeeperMetadataSyncPath(); this.graphMetadataStore = new AccumuloGraphMetadataStore(curatorFramework, zkPath); this.verticesTableName = getVerticesTableName(getConfiguration().getTableNamePrefix()); this.edgesTableName = getEdgesTableName(getConfiguration().getTableNamePrefix()); this.extendedDataTableName = getExtendedDataTableName(getConfiguration().getTableNamePrefix()); this.dataTableName = getDataTableName(getConfiguration().getTableNamePrefix()); this.metadataTableName = getMetadataTableName(getConfiguration().getTableNamePrefix()); this.numberOfQueryThreads = getConfiguration().getNumberOfQueryThreads(); this.historyInSeparateTable = getConfiguration().isHistoryInSeparateTable(); this.historyVerticesTableName = getHistoryVerticesTableName(getConfiguration().getTableNamePrefix()); this.historyEdgesTableName = getHistoryEdgesTableName(getConfiguration().getTableNamePrefix()); } else { this.historyVerticesTableName = null; BatchWriterConfig writerConfig = getConfiguration().createBatchWriterConfig(); this.batchWriter = connector.createMultiTableBatchWriter(writerConfig);
super.setup(context); String configPrefix = context.getConfiguration().get(GRAPH_CONFIG_PREFIX, ""); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(context.getConfiguration(), configPrefix); String tableNamePrefix = accumuloGraphConfiguration.getTableNamePrefix(); final Text edgesTableName = new Text(AccumuloGraph.getEdgesTableName(tableNamePrefix)); final Text dataTableName = new Text(AccumuloGraph.getDataTableName(tableNamePrefix)); VertexiumSerializer vertexiumSerializer = accumuloGraphConfiguration.createSerializer(this.graph); nameSubstitutionStrategy = accumuloGraphConfiguration.createSubstitutionStrategy(this.graph); StreamingPropertyValueStorageStrategy streamingPropertyValueStorageStrategy = accumuloGraphConfiguration.createStreamingPropertyValueStorageStrategy(this.graph); this.elementMutationBuilder = new ElementMutationBuilder(streamingPropertyValueStorageStrategy, vertexiumSerializer) { @Override
public Connector createConnector() { try { LOGGER.info("Connecting to accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()); ZooKeeperInstance instance = new ZooKeeperInstance(getClientConfiguration()); return instance.getConnector(this.getAccumuloUsername(), this.getAuthenticationToken()); } catch (Exception ex) { throw new VertexiumException( String.format("Could not connect to Accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()), ex ); } }
@Test public void testBatchWriterConfigIsSetToValuesWithParameters() { int numThreads = 2; long timeout = 3; long maxMemory = 5; long maxLatency = 7; Map configMap = Maps.newHashMap(); MapUtils.putAll(configMap, new String[]{ AccumuloGraphConfiguration.BATCHWRITER_MAX_LATENCY, "" + maxLatency, AccumuloGraphConfiguration.BATCHWRITER_MAX_MEMORY, "" + maxMemory, AccumuloGraphConfiguration.BATCHWRITER_MAX_WRITE_THREADS, "" + numThreads, AccumuloGraphConfiguration.BATCHWRITER_TIMEOUT, "" + timeout}); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(configMap); BatchWriterConfig batchWriterConfig = accumuloGraphConfiguration.createBatchWriterConfig(); assertThat(batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS), is(maxLatency)); assertThat(batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS), is(timeout)); assertThat(batchWriterConfig.getMaxMemory(), is(maxMemory)); assertThat(batchWriterConfig.getMaxWriteThreads(), is(numThreads)); } }
@SuppressWarnings("unchecked") public ClientConfiguration getClientConfiguration() { ClientConfiguration config = ClientConfiguration.create() .withInstance(this.getAccumuloInstanceName()) .withZkHosts(this.getZookeeperServers()); for (Map.Entry<String, String> entry : getClientConfigurationProperties().entrySet()) { config.setProperty(entry.getKey(), entry.getValue()); } return config; }
BatchWriter writer = graph.getConnector().createBatchWriter( graph.getDataTableName(), graph.getConfiguration().createBatchWriterConfig() ); String lastRowIdPrefix = null;
public static AccumuloGraph create(AccumuloGraphConfiguration config) { if (config == null) { throw new IllegalArgumentException("config cannot be null"); } Connector connector = config.createConnector(); if (config.isHistoryInSeparateTable()) { ensureTableExists(connector, getVerticesTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getEdgesTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getHistoryVerticesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getHistoryEdgesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureRowDeletingIteratorIsAttached(connector, getHistoryVerticesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getHistoryEdgesTableName(config.getTableNamePrefix())); } else { ensureTableExists(connector, getVerticesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getEdgesTableName(config.getTableNamePrefix()), config.getMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); } ensureTableExists(connector, getExtendedDataTableName(config.getTableNamePrefix()), config.getExtendedDataMaxVersions(), config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getDataTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureTableExists(connector, getMetadataTableName(config.getTableNamePrefix()), 1, config.getHdfsContextClasspath(), config.isCreateTables()); ensureRowDeletingIteratorIsAttached(connector, getVerticesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getEdgesTableName(config.getTableNamePrefix())); ensureRowDeletingIteratorIsAttached(connector, getDataTableName(config.getTableNamePrefix())); AccumuloGraph graph = new AccumuloGraph(config, connector); graph.setup(); return graph; }
public static AccumuloGraphConfiguration getAccumuloGraphConfiguration(Configuration conf) { return new AccumuloGraphConfiguration(conf, conf.get("MRMigrationBase.config.prefix")); }