/** * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job. * * @param job * the Hadoop job instance to be configured * @param instanceName * the Accumulo instance name * @param zooKeepers * a comma-separated list of zookeeper servers * @since 1.5.0 * @deprecated since 1.6.0; Use * {@link #setZooKeeperInstance(Job, org.apache.accumulo.core.client.ClientConfiguration)} * instead. */ @Deprecated public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) { setZooKeeperInstance(job, org.apache.accumulo.core.client.ClientConfiguration.create() .withInstance(instanceName).withZkHosts(zooKeepers)); }
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setZooKeeperInstance(job, instanceName, zooKeepers); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
@Override public int run(String[] args) throws Exception { Job job = new Job(getConf(), this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); job.setInputFormatClass(ChunkInputFormat.class); ChunkInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); ChunkInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations(args[5].split(","))); job.getConfiguration().set(VIS, args[6]); job.setMapperClass(HistMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), false, args[4]); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; } }
public static void initSumMRJob(Job job, String inputPath, String outtable, String auths) throws AccumuloSecurityException, IOException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); if (zookeepers != null) { AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify zookeepers"); } SequenceFileInputFormat.addInputPath(job, new Path(inputPath)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(TripleEntry.class); job.setMapOutputValueClass(CardList.class); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
AccumuloOutputFormat.setZooKeeperInstance(conf, args[6], args[7]); AccumuloOutputFormat.setOutputInfo(conf, args[8], args[9].getBytes(), true, null); AccumuloOutputFormat.setMaxMutationBufferSize(conf, 10L * 1000 * 1000);
job.setOutputValueClass(Mutation.class); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), username, password.getBytes(), true, args[3]); AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); job.waitForCompletion(true); return 0;
AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]); AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), true, args[6]);
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = ClientConfiguration.create() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
public static void setOutputInfo(Job job, String instanceName, String zooKeepers, String principal, AuthenticationToken token) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, principal, token); ClientConfiguration clientConfig = ClientConfiguration.create() .withInstance(instanceName) .withZkHosts(zooKeepers); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); }
/** * Configures a {@link ZooKeeperInstance} for this job. * * @param job * the Hadoop job instance to be configured * @param instanceName * the Accumulo instance name * @param zooKeepers * a comma-separated list of zookeeper servers * @since 1.5.0 * @deprecated since 1.6.0; Use {@link #setZooKeeperInstance(Job, ClientConfiguration)} instead. */ @Deprecated public static void setZooKeeperInstance(Job job, String instanceName, String zooKeepers) { setZooKeeperInstance(job, new ClientConfiguration().withInstance(instanceName).withZkHosts(zooKeepers)); }
private static void setAccumuloOutput(final String instStr, final String zooStr, final String userStr, final String passStr, final Job job, final String tableName) throws AccumuloSecurityException { final AuthenticationToken token = new PasswordToken(passStr); AccumuloOutputFormat.setConnectorInfo(job, userStr, token); AccumuloOutputFormat.setDefaultTableName(job, tableName); AccumuloOutputFormat.setCreateTables(job, true); //TODO best way to do this? if (zooStr.equals("mock")) { AccumuloOutputFormat.setMockInstance(job, instStr); } else { AccumuloOutputFormat.setZooKeeperInstance(job, instStr, zooStr); } job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); AccumuloOutputFormat.setZooKeeperInstance(job, this.getClientConfiguration()); }
public static void configureJob(Job job, int numDays, String instance, String zookeepers, String userName, String password, String outputTable) throws AccumuloSecurityException { job.setNumReduceTasks(Math.min(numDays, 100)); // Cap the number of reducers at 100, just in case we have a large day range (shouldn't really happen // though) job.setReducerClass(MetricsDailySummaryReducer.class); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers)); AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(password)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
@Override public void setStoreLocation(final String location, final Job job) throws IOException { conf = job.getConfiguration(); setLocationFromUri(location, job); if (!conf.getBoolean(AccumuloOutputFormat.class.getSimpleName() + ".configured", false)) { try { AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(userP.getBytes(StandardCharsets.UTF_8))); } catch (final AccumuloSecurityException e) { throw new RuntimeException(e); } AccumuloOutputFormat.setDefaultTableName(job, table); AccumuloOutputFormat.setZooKeeperInstance(job, inst, zookeepers); final BatchWriterConfig config = new BatchWriterConfig(); config.setMaxLatency(10, TimeUnit.SECONDS); config.setMaxMemory(10 * 1000 * 1000); config.setMaxWriteThreads(10); AccumuloOutputFormat.setBatchWriterOptions(job, config); } }
public void setStoreLocation(String location, Job job) throws IOException { setLocationFromUri(location); loadDependentJars(job.getConfiguration()); Map<String, String> entries = getOutputFormatEntries(job .getConfiguration()); unsetEntriesFromConfiguration(job.getConfiguration(), entries); try { AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken( password)); } catch (AccumuloSecurityException e) { throw new IOException(e); } AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setZooKeeperInstance(job, inst, zookeepers); BatchWriterConfig bwConfig = new BatchWriterConfig(); bwConfig.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS); bwConfig.setMaxMemory(maxMutationBufferSize); bwConfig.setMaxWriteThreads(maxWriteThreads); AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig); log.info("Writing data to " + table); configureOutputFormat(job); }
/** * Sets up Accumulo output for a job: allows the job to write (String, * Mutation) pairs, where the Mutation will be written to the table named by * the String. * @param job Job to configure * @param outputTable Default table to send output to * @throws AccumuloSecurityException if connecting to Accumulo with the * given username and password fails */ protected void setupAccumuloOutput(Job job, String outputTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); if (mock) { AccumuloOutputFormat.setMockInstance(job, instance); } else { ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); } job.setOutputFormatClass(AccumuloOutputFormat.class); }
/** * Sets up Accumulo output for a job: allows the job to write (String, * Mutation) pairs, where the Mutation will be written to the table named by * the String. * @param job Job to configure * @param outputTable Default table to send output to * @throws AccumuloSecurityException if connecting to Accumulo with the * given username and password fails */ protected void setupAccumuloOutput(Job job, String outputTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); if (mock) { AccumuloOutputFormat.setMockInstance(job, instance); } else { ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); } job.setOutputFormatClass(AccumuloOutputFormat.class); }
public AccumuloConfiguration(Instance instance, String accumuloUser, String accumuloPassword, boolean isMock) throws AccumuloSecurityException, IOException { //NOTE: new Job(new Configuration()) does not work in scala shell due to the toString method's implementation //to get it to work in scala override the toString method and it will work //initialize fields, these are needed for lazy initialization of connector this.zkInstance = instance; this.accumuloUser = accumuloUser; this.accumuloPassword = accumuloPassword; this.job = new Job(new Configuration()); AbstractInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword)); AccumuloOutputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword)); AbstractInputFormat.setScanAuthorizations(job, new Authorizations()); if (isMock) { AbstractInputFormat.setMockInstance(job, instance.getInstanceName()); AccumuloOutputFormat.setMockInstance(job, instance.getInstanceName()); } else { this.clientConfig = new ClientConfiguration(); this.clientConfig.withInstance(instance.getInstanceName()); this.clientConfig.withZkHosts(instance.getZooKeepers()); AbstractInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloOutputFormat.setZooKeeperInstance(job, this.clientConfig); } }