public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(CompositeType.class); job.setMapOutputValueClass(TripleCard.class); // OUTPUT SequenceFileOutputFormat.setOutputPath(job, new Path(outpath)); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(CompositeType.class); job.setOutputValueClass(TripleCard.class); }
@Override public int run(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <table>"); } String user = getAdminPrincipal(); AuthenticationToken pass = getAdminToken(); String table = args[0]; Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloRowInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, user, pass); AccumuloInputFormat.setInputTableName(job, table); AccumuloRowInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "Usage : " + MRTester.class.getName() + " <table> <outputfile>"); } String table = args[0]; assertionErrors.put(table + "_map", new AssertionError("Dummy_map")); assertionErrors.put(table + "_cleanup", new AssertionError("Dummy_cleanup")); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken()); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); AccumuloFileOutputFormat.setOutputPath(job, new Path(args[1])); AccumuloFileOutputFormat.setSampler(job, SAMPLER_CONFIG); job.setMapperClass( table.endsWith("_mapreduce_bad_table") ? BadKeyMapper.class : Mapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); job.getConfiguration().set("MRTester_tableName", table); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
@Override public int run(String[] strings) throws Exception { conf.set(MRUtils.JOB_NAME_PROP, "Upgrade to Rya 3.2.2"); //faster init(); Job job = new Job(conf); job.setJarByClass(Upgrade322Tool.class); setupAccumuloInput(job); AccumuloInputFormat.setInputTableName(job, MRUtils.getTablePrefix(conf) + TBL_OSP_SUFFIX); //we do not need to change any row that is a string, custom, or uri type IteratorSetting regex = new IteratorSetting(30, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, "\\w*" + TYPE_DELIM + "[\u0003|\u0008|\u0002]", null, null, null, false); RegExFilter.setNegate(regex, true); // set input output of the particular job job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); setupAccumuloOutput(job, MRUtils.getTablePrefix(conf) + TBL_SPO_SUFFIX); // set mapper and reducer classes job.setMapperClass(Upgrade322Mapper.class); job.setReducerClass(Reducer.class); // Submit the job return job.waitForCompletion(true) ? 0 : 1; }
@Override public int run(String[] strings) throws Exception { conf.set(MRUtils.JOB_NAME_PROP, "Upgrade to Rya 3.2.2"); //faster init(); Job job = new Job(conf); job.setJarByClass(Upgrade322Tool.class); setupAccumuloInput(job); AccumuloInputFormat.setInputTableName(job, MRUtils.getTablePrefix(conf) + TBL_OSP_SUFFIX); //we do not need to change any row that is a string, custom, or iri type IteratorSetting regex = new IteratorSetting(30, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, "\\w*" + TYPE_DELIM + "[\u0003|\u0008|\u0002]", null, null, null, false); RegExFilter.setNegate(regex, true); // set input output of the particular job job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); setupAccumuloOutput(job, MRUtils.getTablePrefix(conf) + TBL_SPO_SUFFIX); // set mapper and reducer classes job.setMapperClass(Upgrade322Mapper.class); job.setReducerClass(Reducer.class); // Submit the job return job.waitForCompletion(true) ? 0 : 1; }
/** * Set up a MapReduce Job to use Accumulo as input. */ static void configureAccumuloInput(Job job) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(MRUtils.AC_USERNAME_PROP, "root"); String password = conf.get(MRUtils.AC_PWD_PROP, ""); String instance = conf.get(MRUtils.AC_INSTANCE_PROP, "instance"); String zookeepers = conf.get(MRUtils.AC_ZK_PROP, "zoo"); Authorizations auths; String auth = conf.get(MRUtils.AC_AUTH_PROP); if (auth != null) { auths = new Authorizations(auth.split(",")); } else { auths = AccumuloRdfConstants.ALL_AUTHORIZATIONS; } AccumuloInputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zookeepers)); AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setInputTableName(job, getTableName(conf)); AccumuloInputFormat.setScanAuthorizations(job, auths); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
throw new RuntimeException(e); AccumuloInputFormat.setInputTableName(job, table); AccumuloInputFormat.setScanAuthorizations(job, authorizations); if (!mock) {
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure * programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link io.fluo.api.config.FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken(fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final ClientInfo info = getClientInfo(); AccumuloInputFormat.setClientInfo(job, info); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final String principal = getPrincipal(); final AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String principal = getPrincipal(), tableName = getTableName(); if (tokenFile.isEmpty()) { AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setConnectorInfo(job, principal, token); } else { AccumuloInputFormat.setConnectorInfo(job, principal, tokenFile); AccumuloOutputFormat.setConnectorInfo(job, principal, tokenFile); } AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }