public static void initSumMRJob(Job job, String inputPath, String outtable, String auths) throws AccumuloSecurityException, IOException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); if (zookeepers != null) { AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify zookeepers"); } SequenceFileInputFormat.addInputPath(job, new Path(inputPath)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(TripleEntry.class); job.setMapOutputValueClass(CardList.class); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
private static void setAccumuloOutput(final String instStr, final String zooStr, final String userStr, final String passStr, final Job job, final String tableName) throws AccumuloSecurityException { final AuthenticationToken token = new PasswordToken(passStr); AccumuloOutputFormat.setConnectorInfo(job, userStr, token); AccumuloOutputFormat.setDefaultTableName(job, tableName); AccumuloOutputFormat.setCreateTables(job, true); //TODO best way to do this? if (zooStr.equals("mock")) { AccumuloOutputFormat.setMockInstance(job, instStr); } else { AccumuloOutputFormat.setZooKeeperInstance(job, instStr, zooStr); } job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
public static void configureJob(Job job, int numDays, String instance, String zookeepers, String userName, String password, String outputTable) throws AccumuloSecurityException { job.setNumReduceTasks(Math.min(numDays, 100)); // Cap the number of reducers at 100, just in case we have a large day range (shouldn't really happen // though) job.setReducerClass(MetricsDailySummaryReducer.class); job.setOutputFormatClass(AccumuloOutputFormat.class); AccumuloOutputFormat.setZooKeeperInstance(job, ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers)); AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(password)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); }
public static void setOutputInfo(Job job, String user, byte[] passwd, boolean createTables, String defaultTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(passwd)); AccumuloOutputFormat.setCreateTables(job, createTables); AccumuloOutputFormat.setDefaultTableName(job, defaultTable); }
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { Configuration conf = job.getConfiguration(); String username = conf.get(USERNAME); String password = conf.get(PASSWORD); String instance = conf.get(INSTANCE); String zookeepers = conf.get(ZOOKEEPERS); System.out.println("Zookeepers are " + auths); if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); AccumuloInputFormat.setInputTableName(job, intable); job.setInputFormatClass(AccumuloInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // OUTPUT AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); AccumuloOutputFormat.setDefaultTableName(job, outtable); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); }
public static void initMRJob(final Job job, final String table, final String outtable, final String[] auths) throws AccumuloSecurityException { final Configuration conf = job.getConfiguration(); final String username = conf.get(USERNAME); final String password = conf.get(PASSWORD); final String instance = conf.get(INSTANCE); final String zookeepers = conf.get(ZOOKEEPERS); final String mock = conf.get(MOCK); //input if (Boolean.parseBoolean(mock)) { AccumuloInputFormat.setMockInstance(job, instance); AccumuloOutputFormat.setMockInstance(job, instance); } else if (zookeepers != null) { AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); } else { throw new IllegalArgumentException("Must specify either mock or zookeepers"); } AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloInputFormat.setInputTableName(job, table); job.setInputFormatClass(AccumuloInputFormat.class); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); // OUTPUT job.setOutputFormatClass(AccumuloOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8))); AccumuloOutputFormat.setDefaultTableName(job, outtable); }
AccumuloOutputFormat.setDefaultTableName(job, table2); AccumuloOutputFormat.setZooKeeperInstance(job, getCluster().getClientConfig());
AccumuloOutputFormat.setDefaultTableName(job, table2); AccumuloOutputFormat.setZooKeeperInstance(job, getCluster().getClientConfig());
@Override public void setStoreLocation(final String location, final Job job) throws IOException { conf = job.getConfiguration(); setLocationFromUri(location, job); if (!conf.getBoolean(AccumuloOutputFormat.class.getSimpleName() + ".configured", false)) { try { AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(userP.getBytes(StandardCharsets.UTF_8))); } catch (final AccumuloSecurityException e) { throw new RuntimeException(e); } AccumuloOutputFormat.setDefaultTableName(job, table); AccumuloOutputFormat.setZooKeeperInstance(job, inst, zookeepers); final BatchWriterConfig config = new BatchWriterConfig(); config.setMaxLatency(10, TimeUnit.SECONDS); config.setMaxMemory(10 * 1000 * 1000); config.setMaxWriteThreads(10); AccumuloOutputFormat.setBatchWriterOptions(job, config); } }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, this.principal, this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { AccumuloInputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloOutputFormat.setConnectorInfo(job, getPrincipal(), this.getToken()); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, this.auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); job.getConfiguration().set("MRMigrationBase.config.prefix", configPrefix); job.getConfiguration().set("MRMigrationBase.output.tableName", getTableName()); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); AccumuloInputFormat.setInputTableName(job, getTableName()); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, getTableName()); }
/** * Sets up Accumulo output for a job: allows the job to write (String, * Mutation) pairs, where the Mutation will be written to the table named by * the String. * @param job Job to configure * @param outputTable Default table to send output to * @throws AccumuloSecurityException if connecting to Accumulo with the * given username and password fails */ protected void setupAccumuloOutput(Job job, String outputTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); if (mock) { AccumuloOutputFormat.setMockInstance(job, instance); } else { ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); } job.setOutputFormatClass(AccumuloOutputFormat.class); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final ClientInfo info = getClientInfo(); AccumuloInputFormat.setClientInfo(job, info); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
/** * Sets up Accumulo output for a job: allows the job to write (String, * Mutation) pairs, where the Mutation will be written to the table named by * the String. * @param job Job to configure * @param outputTable Default table to send output to * @throws AccumuloSecurityException if connecting to Accumulo with the * given username and password fails */ protected void setupAccumuloOutput(Job job, String outputTable) throws AccumuloSecurityException { AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd)); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, outputTable); if (mock) { AccumuloOutputFormat.setMockInstance(job, instance); } else { ClientConfiguration clientConfig = ClientConfiguration.loadDefault() .withInstance(instance).withZkHosts(zk); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); } job.setOutputFormatClass(AccumuloOutputFormat.class); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String principal = getPrincipal(), tableName = getTableName(); if (tokenFile.isEmpty()) { AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setConnectorInfo(job, principal, token); } else { AccumuloInputFormat.setConnectorInfo(job, principal, tokenFile); AccumuloOutputFormat.setConnectorInfo(job, principal, tokenFile); } AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
@Override public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { super.setAccumuloConfigs(job); final String tableName = getTableName(); final String principal = getPrincipal(); final AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); AccumuloInputFormat.setInputTableName(job, tableName); AccumuloInputFormat.setScanAuthorizations(job, auths); AccumuloOutputFormat.setConnectorInfo(job, principal, token); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, tableName); }
AccumuloOutputFormat.setDefaultTableName(job, tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX); if (mock) { RyaOutputFormat.setMockInstance(job, instance);
AccumuloOutputFormat.setDefaultTableName(job, tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX); if (mock) { RyaOutputFormat.setMockInstance(job, instance);