public AccumuloElementInputFormatBase() { accumuloInputFormat = new AccumuloRowInputFormat(); }
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException { return accumuloInputFormat.getSplits(jobContext); }
@Override public RecordReader<Text,PeekingIterator<Entry<Key,Value>>> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { log.setLevel(getLogLevel(context)); return new RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>>() { RowIterator rowIterator; @Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); currentV = null; } @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!rowIterator.hasNext()) return false; currentV = new PeekingIterator<>(rowIterator.next()); numKeysRead = rowIterator.getKVCount(); currentKey = currentV.peek().getKey(); currentK = new Text(currentKey.getRow()); return true; } }; } }
protected static void setInputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token, String[] authorizations, String tableName) throws AccumuloSecurityException { AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); AccumuloRowInputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(VertexiumMRUtils.CONFIG_AUTHORIZATIONS, authorizations); }
@Override public RecordReader<Text, TValue> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final RecordReader<Text, PeekingIterator<Map.Entry<Key, Value>>> reader = accumuloInputFormat.createRecordReader(inputSplit, taskAttemptContext); return new RecordReader<Text, TValue>() { public AccumuloGraph graph;
@Override public int run(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <table>"); } String user = getAdminPrincipal(); AuthenticationToken pass = getAdminToken(); String table = args[0]; Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloRowInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, user, pass); AccumuloInputFormat.setInputTableName(job, table); AccumuloRowInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
protected static void setInputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token, String[] authorizations, String tableName) throws AccumuloSecurityException { AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); AccumuloRowInputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(SecureGraphMRUtils.CONFIG_AUTHORIZATIONS, authorizations); }
@Override public RecordReader<Text, TValue> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final RecordReader<Text, PeekingIterator<Map.Entry<Key, Value>>> reader = accumuloInputFormat.createRecordReader(inputSplit, taskAttemptContext); return new RecordReader<Text, TValue>() { public AccumuloGraph graph;
public static void setInputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token, String[] authorizations, String tableName) throws AccumuloSecurityException { AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = ClientConfiguration.create() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloRowInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(VertexiumMRUtils.CONFIG_AUTHORIZATIONS, authorizations); }
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { return accumuloInputFormat.getSplits(jobContext); }
public AccumuloElementInputFormatBase() { accumuloInputFormat = new AccumuloRowInputFormat(); }
@Override public RecordReader<Text, TValue> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final RecordReader<Text, PeekingIterator<Map.Entry<Key, Value>>> reader = accumuloInputFormat.createRecordReader(inputSplit, taskAttemptContext); return new RecordReader<Text, TValue>() { public AccumuloGraph graph;
@Override public RecordReader<Text,PeekingIterator<Entry<Key,Value>>> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { log.setLevel(getLogLevel(context)); return new RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>>() { RowIterator rowIterator; @Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); currentV = null; } @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!rowIterator.hasNext()) return false; currentV = new PeekingIterator<>(rowIterator.next()); numKeysRead = rowIterator.getKVCount(); currentKey = currentV.peek().getKey(); currentK = new Text(currentKey.getRow()); return true; } }; } }
public static void setInputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token, String[] authorizations, String tableName) throws AccumuloSecurityException { AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = ClientConfiguration.create() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloRowInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(VertexiumMRUtils.CONFIG_AUTHORIZATIONS, authorizations); }
@Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { return this.accumuloInputFormat.getSplits(context); }
public AccumuloElementInputFormatBase() { accumuloInputFormat = new AccumuloRowInputFormat(); }
@Override public RecordReader<Text, TValue> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final RecordReader<Text, PeekingIterator<Map.Entry<Key, Value>>> reader = accumuloInputFormat.createRecordReader(inputSplit, taskAttemptContext); return new RecordReader<Text, TValue>() { public AccumuloGraph graph;
@Override public RecordReader<Text,PeekingIterator<Entry<Key,Value>>> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { log.setLevel(getLogLevel(context)); return new RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>>() { RowIterator rowIterator; @Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); currentV = null; } @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!rowIterator.hasNext()) return false; currentV = new PeekingIterator<>(rowIterator.next()); numKeysRead = rowIterator.getKVCount(); currentKey = currentV.peek().getKey(); currentK = new Text(currentKey.getRow()); return true; } }; } }
public static void configure(Job job) { try { AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(job.getConfiguration(), "graph."); AccumuloGraph graph = AccumuloGraph.create(accumuloGraphConfiguration); String principal = accumuloGraphConfiguration.getAccumuloUsername(); AuthenticationToken token = accumuloGraphConfiguration.getAuthenticationToken(); String instanceName = accumuloGraphConfiguration.getAccumuloInstanceName(); String zooKeepers = accumuloGraphConfiguration.getZookeeperServers(); String[] authorizations = job.getConfiguration().get("titan.hadoop.input.authorizations", "").split(","); if (authorizations.length == 1 && authorizations[0].trim().length() == 0) { authorizations = new String[0]; } String tableName = graph.getVerticesTableName(); AccumuloRowInputFormat.setInputTableName(job, tableName); AccumuloRowInputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = new ClientConfiguration() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloRowInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloRowInputFormat.setScanAuthorizations(job, new org.apache.accumulo.core.security.Authorizations(authorizations)); job.getConfiguration().setStrings(VertexiumMRUtils.CONFIG_AUTHORIZATIONS, authorizations); } catch (Exception ex) { throw new VertexiumException("Could not configure", ex); } }
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException { return accumuloInputFormat.getSplits(jobContext); }