public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(CompositeType.class); job.setMapOutputValueClass(TripleCard.class); // OUTPUT SequenceFileOutputFormat.setOutputPath(job, new Path(outpath)); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(CompositeType.class); job.setOutputValueClass(TripleCard.class); }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
public static void initMRJob(final Job job, final String table, final String outtable, final String[] auths) throws AccumuloSecurityException { final Configuration conf = job.getConfiguration(); final String username = conf.get(USERNAME); final String password = conf.get(PASSWORD); final String instance = conf.get(INSTANCE); final String zookeepers = conf.get(ZOOKEEPERS); final String mock = conf.get(MOCK); //input if (Boolean.parseBoolean(mock)) { AccumuloInputFormat.setMockInstance(job, instance); AccumuloOutputFormat.setMockInstance(job, instance); } else if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloInputFormat.setInputTableName(job, table); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); // OUTPUT job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloOutputFormat.setDefaultTableName(job, outtable); }
/** * Set up a MapReduce Job to use Accumulo as input. */ static void configureAccumuloInput(Job job) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(MRUtils.AC_USERNAME_PROP, "root"); String password = conf.get(MRUtils.AC_PWD_PROP, ""); String instance = conf.get(MRUtils.AC_INSTANCE_PROP, "instance"); String zookeepers = conf.get(MRUtils.AC_ZK_PROP, "zoo"); Authorizations auths; String auth = conf.get(MRUtils.AC_AUTH_PROP); if (auth != null) { auths = new Authorizations(auth.split(",")); } else { auths = AccumuloRdfConstants.ALL_AUTHORIZATIONS; } AccumuloInputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zookeepers)); AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setInputTableName(job, getTableName(conf)); AccumuloInputFormat.setScanAuthorizations(job, auths); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
AccumuloInputFormat.setScanAuthorizations(job, authorizations); if (!mock) { AccumuloInputFormat.setZooKeeperInstance(job, inst, zookeepers);
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure * programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link io.fluo.api.config.FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken(fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure * programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken(fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, authorizations); if (mock) { AccumuloInputFormat.setMockInstance(job, instance);
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final ClientInfo info = getClientInfo(); AccumuloInputFormat.setClientInfo(job, info); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String principal = getPrincipal(), tableName = getTableName(); if (tokenFile.isEmpty()) { AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setConnectorInfo(job, principal, token); } else { AccumuloInputFormat.setConnectorInfo(job, principal, tokenFile); AccumuloOutputFormat.setConnectorInfo(job, principal, tokenFile); } AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final String principal = getPrincipal(); final AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloInputFormat.setScanIsolation(job, isolated); AccumuloInputFormat.setLocalIterators(job, localIters);