public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(CompositeType.class); job.setMapOutputValueClass(TripleCard.class); // OUTPUT SequenceFileOutputFormat.setOutputPath(job, new Path(outpath)); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(CompositeType.class); job.setOutputValueClass(TripleCard.class); }
public static void setFetchHints(Job job, ElementType elementType, FetchHints fetchHints) { Iterable<Text> columnFamiliesToFetch = AccumuloGraph.getColumnFamiliesToFetch(elementType, fetchHints); Collection<Pair<Text, Text>> columnFamilyColumnQualifierPairs = new ArrayList<>(); for (Text columnFamilyToFetch : columnFamiliesToFetch) { columnFamilyColumnQualifierPairs.add(new Pair<>(columnFamilyToFetch, null)); } AccumuloInputFormat.fetchColumns(job, columnFamilyColumnQualifierPairs); }
@Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { return new AccumuloInputFormat().getSplits(context); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final String principal = getPrincipal(); final AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
connector.tableOperations().create(table); AccumuloInputFormat.setZooKeeperInstance(job, cluster.getClientConfig()); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloInputFormat.setScanIsolation(job, isolated); AccumuloInputFormat.setLocalIterators(job, localIters); AccumuloInputFormat.fetchColumns(job, fetchColumns); AccumuloInputFormat.setLogLevel(job, level); AccumuloInputFormat aif = new AccumuloInputFormat(); List<InputSplit> splits = aif.getSplits(job);
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "Usage : " + MRTester.class.getName() + " <table> <outputfile>"); } String table = args[0]; assertionErrors.put(table + "_map", new AssertionError("Dummy_map")); assertionErrors.put(table + "_cleanup", new AssertionError("Dummy_cleanup")); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); AccumuloFileOutputFormat.setOutputPath(job, new Path(args[1])); AccumuloFileOutputFormat.setSampler(job, SAMPLER_CONFIG); job.setMapperClass( table.endsWith("_mapreduce_bad_table") ? BadKeyMapper.class : Mapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); job.getConfiguration().set("MRTester_tableName", table); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(password)); AccumuloInputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers)); AccumuloInputFormat.setRanges(job, dayRanges); AccumuloInputFormat.setAutoAdjustRanges(job, false); AccumuloInputFormat.setInputTableName(job, inputTable); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloInputFormat.addIterator(job, regex);
AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setZooKeeperInstance(job, clientConf); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); List<InputSplit> splits = inputFormat.getSplits(job); assertEquals(actualSplits.size() + 1, splits.size()); // No ranges set on the job so it'll start for (Text text : actualSplits) ranges.add(new Range(text)); AccumuloInputFormat.setRanges(job, ranges); splits = inputFormat.getSplits(job); assertEquals(actualSplits.size(), splits.size()); AccumuloInputFormat.setOfflineTableScan(job, true); try { inputFormat.getSplits(job); fail("An exception should have been thrown"); } catch (IOException e) {} splits = inputFormat.getSplits(job); assertEquals(actualSplits.size(), splits.size()); AccumuloInputFormat.setRanges(job, ranges); splits = inputFormat.getSplits(job); assertEquals(2, splits.size()); AccumuloInputFormat.setAutoAdjustRanges(job, false); splits = inputFormat.getSplits(job); assertEquals(ranges.size(), splits.size());
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final ClientInfo info = getClientInfo(); AccumuloInputFormat.setClientInfo(job, info); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
AccumuloInputFormat.setZooKeeperInstance(job, cluster.getClientConfig()); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setBatchScan(job, batchScan); if (sample) { AccumuloInputFormat.setSamplerConfiguration(job, SAMPLER_CONFIG);
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); }
@Override public int run(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <table>"); } String user = getAdminPrincipal(); AuthenticationToken pass = getAdminToken(); String table = args[0]; Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloRowInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, user, pass); AccumuloInputFormat.setInputTableName(job, table); AccumuloRowInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations()); AccumuloInputFormat.fetchColumns(job.getConfiguration(), columnsToFetch);
opts.maxMaps); conn.tableOperations().offline(clone); AccumuloInputFormat.setInputTableName(job, clone); AccumuloInputFormat.setOfflineTableScan(job, true); } else { ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false);
public int run(String[] args) throws Exception { Job job = new Job(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations()); IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, args[5], args[6], args[7], args[8], false); AccumuloInputFormat.addIterator(job.getConfiguration(), regex); job.setMapperClass(RegexMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setNumReduceTasks(0); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(args[9])); System.out.println("setRowRegex: " + args[5]); System.out.println("setColumnFamilyRegex: " + args[6]); System.out.println("setColumnQualifierRegex: " + args[7]); System.out.println("setValueRegex: " + args[8]); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
@Override public int run(String[] strings) throws Exception { conf.set(MRUtils.JOB_NAME_PROP, "Upgrade to Rya 3.2.2"); //faster init(); Job job = new Job(conf); job.setJarByClass(Upgrade322Tool.class); setupAccumuloInput(job); AccumuloInputFormat.setInputTableName(job, MRUtils.getTablePrefix(conf) + TBL_OSP_SUFFIX); //we do not need to change any row that is a string, custom, or uri type IteratorSetting regex = new IteratorSetting(30, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, "\\w*" + TYPE_DELIM + "[\u0003|\u0008|\u0002]", null, null, null, false); RegExFilter.setNegate(regex, true); // set input output of the particular job job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); setupAccumuloOutput(job, MRUtils.getTablePrefix(conf) + TBL_SPO_SUFFIX); // set mapper and reducer classes job.setMapperClass(Upgrade322Mapper.class); job.setReducerClass(Reducer.class); // Submit the job return job.waitForCompletion(true) ? 0 : 1; }
conn.tableOperations().offline(clone); AccumuloInputFormat.setScanOffline(job.getConfiguration(), true); AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), clone, new Authorizations());
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); AccumuloOutputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); }
@Override @SuppressWarnings("rawtypes") public InputFormat getInputFormat() { return new AccumuloInputFormat(); }
Set<Range> ranges = clientOpts.getConnector().tableOperations() .splitRangeByTablets(clientOpts.getTableName(), new Range(), opts.maxMaps); AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); } catch (Exception e) { throw new IOException(e);