private static void setAccumuloOutput(final String instStr, final String zooStr, final String userStr, final String passStr, final Job job, final String tableName) throws AccumuloSecurityException { final AuthenticationToken token = new PasswordToken(passStr); AccumuloOutputFormat.setConnectorInfo(job, userStr, token); AccumuloOutputFormat.setDefaultTableName(job, tableName); AccumuloOutputFormat.setCreateTables(job, true); //TODO best way to do this? if (zooStr.equals("mock")) { AccumuloOutputFormat.setMockInstance(job, instStr); } else { AccumuloOutputFormat.setZooKeeperInstance(job, instStr, zooStr); } job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
@Override public RecordWriter<Text, Mutation> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { return accumuloOutputFormat.getRecordWriter(context); }
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
public static void setOutputInfo(Job job, String user, byte[] passwd, boolean createTables, String defaultTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(passwd)); AccumuloOutputFormat.setCreateTables(job, createTables); AccumuloOutputFormat.setDefaultTableName(job, defaultTable); }
public static void initSumMRJob(Job job, String inputPath, String outtable, String auths) throws AccumuloSecurityException, IOException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); if (zookeepers != null) { AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify zookeepers"); } SequenceFileInputFormat.addInputPath(job, new Path(inputPath)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(TripleEntry.class); job.setMapOutputValueClass(CardList.class); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
AccumuloOutputFormat.setZooKeeperInstance(job, opts.instance, opts.zookeepers); AccumuloOutputFormat.setConnectorInfo(job, opts.principal, opts.getToken()); AccumuloOutputFormat.setCreateTables(job, false);
public AccumuloConfiguration(Instance instance, String accumuloUser, String accumuloPassword, boolean isMock) throws AccumuloSecurityException, IOException { //NOTE: new Job(new Configuration()) does not work in scala shell due to the toString method's implementation //to get it to work in scala override the toString method and it will work //initialize fields, these are needed for lazy initialization of connector this.zkInstance = instance; this.accumuloUser = accumuloUser; this.accumuloPassword = accumuloPassword; this.job = new Job(new Configuration()); AbstractInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword)); AccumuloOutputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword)); AbstractInputFormat.setScanAuthorizations(job, new Authorizations()); if (isMock) { AbstractInputFormat.setMockInstance(job, instance.getInstanceName()); AccumuloOutputFormat.setMockInstance(job, instance.getInstanceName()); } else { this.clientConfig = new ClientConfiguration(); this.clientConfig.withInstance(instance.getInstanceName()); this.clientConfig.withZkHosts(instance.getZooKeepers()); AbstractInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloOutputFormat.setZooKeeperInstance(job, this.clientConfig); } }
@Override public int run(String[] args) throws Exception { Job job = new Job(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); job.setInputFormatClass(ChunkInputFormat.class); ChunkInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); ChunkInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations(args[5].split(","))); job.getConfiguration().set(VIS, args[6]); job.setMapperClass(HistMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), false, args[4]); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; } }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final ClientInfo info = getClientInfo(); AccumuloInputFormat.setClientInfo(job, info); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), true, null); job.setOutputFormatClass(AccumuloOutputFormat.class); RecordWriter<Text,Mutation> rw = new AccumuloOutputFormat().getRecordWriter(context);
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); }
/** * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job. * * @param job * the Hadoop job instance to be configured * @param instanceName * the Accumulo instance name * @param zooKeepers * a comma-separated list of zookeeper servers * @since 1.5.0 * @deprecated since 1.6.0; Use * {@link #setZooKeeperInstance(Job, org.apache.accumulo.core.client.ClientConfiguration)} * instead. */ @Deprecated public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) { setZooKeeperInstance(job, org.apache.accumulo.core.client.ClientConfiguration.create() .withInstance(instanceName).withZkHosts(zooKeepers)); }
AccumuloOutputFormat.setZooKeeperInstance(conf, args[6], args[7]); AccumuloOutputFormat.setOutputInfo(conf, args[8], args[9].getBytes(), true, null); AccumuloOutputFormat.setMaxMutationBufferSize(conf, 10L * 1000 * 1000);
/** * Configure a job to use a mock Accumulo instance. * @param job Job to configure * @param instance Name of the mock instance */ public static void setMockInstance(final Job job, final String instance) { AccumuloOutputFormat.setMockInstance(job, instance); job.getConfiguration().setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); job.getConfiguration().setBoolean(MRUtils.AC_MOCK_PROP, true); }
AccumuloOutputFormat.setBatchWriterOptions(job, bwOpts.getBatchWriterConfig());
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), username, password.getBytes(), true, args[3]); AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); job.waitForCompletion(true); return 0;
/** * Configures a {@link ZooKeeperInstance} for this job. * * @param job * the Hadoop job instance to be configured * @param instanceName * the Accumulo instance name * @param zooKeepers * a comma-separated list of zookeeper servers * @since 1.5.0 * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(Job, ClientConfiguration)} instead. */ @Deprecated public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) { setZooKeeperInstance(job, new ClientConfiguration().withInstance(instanceName).withZkHosts(zooKeepers)); }