@Test public void testWordCount() throws Exception { // TODO Figure out a way to run M/R with Kerberos assumeTrue(getAdminToken() instanceof PasswordToken); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); is = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new Text("count")))); SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING); c.tableOperations().attachIterator(tableName, is); Path readme = new Path(new Path(System.getProperty("user.dir")).getParent(), "README.md"); if (!new File(readme.toString()).exists()) { log.info("Not running test: README.md does not exist)"); return; } fs.copyFromLocalFile(readme, new Path(dir + "/tmp/wc/README.md")); String[] args; if (saslEnabled) { args = new String[] {"-i", instance, "-u", user, "--keytab", keytab, "-z", keepers, "--input", dir + "/tmp/wc", "-t", tableName}; } else { args = new String[] {"-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", tableName}; } goodExec(WordCount.class, args); }
@Override public void visit(State state, Environment env, Properties props) throws Exception { Random rand = new Random(); String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_"); String pid = env.getPid(); tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis()); log.info("Starting bulk test on " + tableName); TableOperations tableOps = env.getConnector().tableOperations(); try { if (!tableOps.exists(getTableName())) { tableOps.create(getTableName()); IteratorSetting is = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING); SummingCombiner.setCombineAllColumns(is, true); tableOps.attachIterator(getTableName(), is); } } catch (TableExistsException ex) { // expected if there are multiple walkers } state.set("rand", rand); state.set("fs", FileSystem.get(CachedConfiguration.getInstance())); state.set("bulkImportSuccess", "true"); BulkPlusOne.counter.set(0l); ThreadPoolExecutor e = new SimpleThreadPool(MAX_POOL_SIZE, "bulkImportPool"); state.set("pool", e); }
IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setColumns(setting, Collections.singletonList(new Column("f"))); SummingCombiner.setEncodingType(setting, SummingCombiner.Type.VARLEN); tops.attachIterator(metadataTableName, setting, EnumSet.allOf(IteratorScope.class));
IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setColumns(setting, Collections.singletonList(new Column("f"))); SummingCombiner.setEncodingType(setting, SummingCombiner.Type.VARLEN); tops.attachIterator(metadataTableName, setting, EnumSet.allOf(IteratorScope.class));
@Test public void aggregationTest() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setEncodingType(setting, Type.STRING); SummingCombiner.setColumns(setting, Collections.singletonList(new IteratorSetting.Column("cf"))); c.tableOperations().attachIterator(tableName, setting); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { Mutation m = new Mutation("row1"); m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes()); bw.addMutation(m); } bw.close(); checkSum(tableName, c); }
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); parseOptions(options); parseScope(env); summer = new SummingCombiner(); IteratorSetting itset = new IteratorSetting(1, SummingCombiner.class); SummingCombiner.setCombineAllColumns(itset, true); SummingCombiner.setEncodingType(itset, LongCombiner.Type.STRING); summer.init(null, itset.getOptions(), env); }
private void createSeqTable(String table) { // create tables if they don't exist if (!connector.tableOperations().exists(table)) { IteratorSetting longCombiner = new IteratorSetting(1, SummingCombiner.class); SummingCombiner.setCombineAllColumns(longCombiner, true); SummingCombiner.setEncodingType(longCombiner, LongLexicoderTemp.class); try { connector.tableOperations().create(table); connector.tableOperations().addSplits(table, getSplitPoints(3)); GraphuloUtil.applyIteratorSoft(longCombiner, connector.tableOperations(), table); } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException | TableExistsException e) { log.warn("", e); } } }
IteratorSetting is = new IteratorSetting(1, SummingCombiner.class); SummingCombiner.setCombineAllColumns(is, true); SummingCombiner.setEncodingType(is, Type.STRING); GraphuloUtil.applyIteratorSoft(is, connector.tableOperations(), countDegree);
SummingCombiner.setEncodingType(iterConfig, Type.STRING); SummingCombiner.setColumns(iterConfig, Collections.singletonList(new IteratorSetting.Column("count"))); SummingCombiner.setEncodingType(iterConfig2, Type.STRING); SummingCombiner.setColumns(iterConfig2, Collections.singletonList(new IteratorSetting.Column("count2", "comments")));
SummingCombiner.setEncodingType(summingCombiner, Type.STRING); SummingCombiner.setCombineAllColumns(summingCombiner, true);