AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations());
public int run(String[] args) throws Exception { Job job = new Job(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations()); IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, args[5], args[6], args[7], args[8], false); AccumuloInputFormat.addIterator(job.getConfiguration(), regex); job.setMapperClass(RegexMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setNumReduceTasks(0); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(args[9])); System.out.println("setRowRegex: " + args[5]); System.out.println("setColumnFamilyRegex: " + args[6]); System.out.println("setColumnQualifierRegex: " + args[7]); System.out.println("setValueRegex: " + args[8]); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations());
public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(CompositeType.class); job.setMapOutputValueClass(TripleCard.class); // OUTPUT SequenceFileOutputFormat.setOutputPath(job, new Path(outpath)); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(CompositeType.class); job.setOutputValueClass(TripleCard.class); }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); AccumuloOutputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "Usage : " + MRTester.class.getName() + " <table> <outputfile>"); } String table = args[0]; assertionErrors.put(table + "_map", new AssertionError("Dummy_map")); assertionErrors.put(table + "_cleanup", new AssertionError("Dummy_cleanup")); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); AccumuloFileOutputFormat.setOutputPath(job, new Path(args[1])); AccumuloFileOutputFormat.setSampler(job, SAMPLER_CONFIG); job.setMapperClass( table.endsWith("_mapreduce_bad_table") ? BadKeyMapper.class : Mapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); job.getConfiguration().set("MRTester_tableName", table); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
AccumuloInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig());
AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), clone, new Authorizations());
public static void initMRJob(final Job job, final String table, final String outtable, final String[] auths) throws AccumuloSecurityException { final Configuration conf = job.getConfiguration(); final String username = conf.get(USERNAME); final String password = conf.get(PASSWORD); final String instance = conf.get(INSTANCE); final String zookeepers = conf.get(ZOOKEEPERS); final String mock = conf.get(MOCK); //input if (Boolean.parseBoolean(mock)) { AccumuloInputFormat.setMockInstance(job, instance); AccumuloOutputFormat.setMockInstance(job, instance); } else if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloInputFormat.setInputTableName(job, table); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); // OUTPUT job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloOutputFormat.setDefaultTableName(job, outtable); }
AccumuloInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig());
/** * Set up a MapReduce Job to use Accumulo as input. */ static void configureAccumuloInput(Job job) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(MRUtils.AC_USERNAME_PROP, "root"); String password = conf.get(MRUtils.AC_PWD_PROP, ""); String instance = conf.get(MRUtils.AC_INSTANCE_PROP, "instance"); String zookeepers = conf.get(MRUtils.AC_ZK_PROP, "zoo"); Authorizations auths; String auth = conf.get(MRUtils.AC_AUTH_PROP); if (auth != null) { auths = new Authorizations(auth.split(",")); } else { auths = AccumuloRdfConstants.ALL_AUTHORIZATIONS; } AccumuloInputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zookeepers)); AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setInputTableName(job, getTableName(conf)); AccumuloInputFormat.setScanAuthorizations(job, auths); }
AccumuloInputFormat.setScanAuthorizations(job, authorizations); if (!mock) { AccumuloInputFormat.setZooKeeperInstance(job, inst, zookeepers); } else { AccumuloInputFormat.setMockInstance(job, inst);
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link io.fluo.api.config.FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken(fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure * programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken(fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
connector.tableOperations().create(table); AccumuloInputFormat.setZooKeeperInstance(job, cluster.getClientConfig()); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken());