String tableName = getInputTableName(implementingClass, conf); if (tableName != null) { org.apache.accumulo.core.client.mapreduce.InputTableConfig queryConfig = new org.apache.accumulo.core.client.mapreduce.InputTableConfig(); List<IteratorSetting> itrs = getIterators(implementingClass, conf); if (itrs != null) queryConfig.setIterators(itrs); Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf); if (columns != null) queryConfig.fetchColumns(columns); List<Range> ranges = null; try { queryConfig.setRanges(ranges); queryConfig.setSamplerConfiguration(samplerConfig); queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)) .setUseIsolatedScanners(isIsolated(implementingClass, conf)) .setUseLocalIterators(usesLocalIterators(implementingClass, conf)) .setOfflineScan(isOfflineScan(implementingClass, conf)); return Maps.immutableEntry(tableName, queryConfig);
org.apache.accumulo.core.client.mapreduce.InputTableConfig tableConfig = tableConfigEntry .getValue(); if (!tableConfig.shouldUseLocalIterators()) { if (tableConfig.getIterators() != null) { for (IteratorSetting iter : tableConfig.getIterators()) { if (!client.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
/** * Central place to set common split configuration not handled by split constructors. The * intention is to make it harder to miss optional setters in future refactor. */ public static void updateSplit(org.apache.accumulo.core.client.mapreduce.RangeInputSplit split, org.apache.accumulo.core.client.mapreduce.InputTableConfig tableConfig, Level logLevel) { split.setFetchedColumns(tableConfig.getFetchedColumns()); split.setIterators(tableConfig.getIterators()); split.setLogLevel(logLevel); split.setSamplerConfiguration(tableConfig.getSamplerConfiguration()); }
isOffline = tableConfig.isOfflineScan(); isIsolated = tableConfig.shouldUseIsolatedScanners(); usesLocalIterators = tableConfig.shouldUseLocalIterators(); columns = tableConfig.getFetchedColumns(); samplerConfig = tableConfig.getSamplerConfiguration();
boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators()); if (batchScan && !supportBatchScan) throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators"); boolean autoAdjust = tableConfig.shouldAutoAdjustRanges(); if (batchScan && !autoAdjust) throw new IllegalArgumentException( "AutoAdjustRanges must be enabled when using BatchScanner optimization"); List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges(); if (ranges.isEmpty()) { ranges = new ArrayList<>(1); TabletLocator tl; try { if (tableConfig.isOfflineScan()) { binnedRanges = binOfflineTable(job, tableId, ranges); while (binnedRanges == null) { org.apache.accumulo.core.clientImpl.mapreduce.SplitUtils.updateSplit(split, tableConfig, logLevel); split.setOffline(tableConfig.isOfflineScan()); split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners()); split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators()); splits.add(split); } else {
/** * Central place to set common split configuration not handled by split constructors. The * intention is to make it harder to miss optional setters in future refactor. */ public static void updateSplit(RangeInputSplit split, InputTableConfig tableConfig, Level logLevel) { split.setFetchedColumns(tableConfig.getFetchedColumns()); split.setIterators(tableConfig.getIterators()); split.setLogLevel(logLevel); split.setSamplerConfiguration(tableConfig.getSamplerConfiguration()); split.setExecutionHints(tableConfig.getExecutionHints()); }
/** * Get table names and input configurations for each range * @return A Map representing each table and {@link InputTableConfig} needed to get all the rows that match the rules. */ public Map<String, InputTableConfig> getInputConfigs() { final Map<String, InputTableConfig> configs = new HashMap<>(); for (final TABLE_LAYOUT layout : tableRanges.keySet()) { final String parentTable = RdfCloudTripleStoreUtils.layoutPrefixToTable(layout, conf.getTablePrefix()); final InputTableConfig config = new InputTableConfig(); config.setRanges(tableRanges.get(layout)); configs.put(parentTable, config); } for (final String tableName : entireTables) { final InputTableConfig config = new InputTableConfig(); final List<Range> ranges = new LinkedList<>(); ranges.add(new Range()); config.setRanges(ranges); configs.put(tableName, config); } return configs; }
final List<IteratorSetting> iterators = entry.getValue().getIterators(); iterators.addAll(additionalSettings); entry.getValue().setIterators(iterators);
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "Usage : " + MRTester.class.getName() + " <table1> <table2>"); } String user = getAdminPrincipal(); AuthenticationToken pass = getAdminToken(); String table1 = args[0]; String table2 = args[1]; Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloMultiTableInputFormat.class); AccumuloMultiTableInputFormat.setConnectorInfo(job, user, pass); InputTableConfig tableConfig1 = new InputTableConfig(); InputTableConfig tableConfig2 = new InputTableConfig(); Map<String,InputTableConfig> configMap = new HashMap<>(); configMap.put(table1, tableConfig1); configMap.put(table2, tableConfig2); AccumuloMultiTableInputFormat.setInputTableConfigs(job, configMap); AccumuloMultiTableInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
@Override protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) { return getInputTableConfig(context, tableName).getIterators(); } };
/** * Creates a batch scan config object out of a previously serialized batch scan config object. * * @param input * the data input of the serialized batch scan config */ public InputTableConfig(DataInput input) throws IOException { readFields(input); }
Boolean isOffline = baseSplit.isOffline(); if (isOffline == null) { isOffline = tableConfig.isOfflineScan(); isIsolated = tableConfig.shouldUseIsolatedScanners(); usesLocalIterators = tableConfig.shouldUseLocalIterators(); columns = tableConfig.getFetchedColumns(); samplerConfig = tableConfig.getSamplerConfiguration();
boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators()); if (batchScan && !supportBatchScan) throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators"); boolean autoAdjust = tableConfig.shouldAutoAdjustRanges(); if (batchScan && !autoAdjust) throw new IllegalArgumentException( "AutoAdjustRanges must be enabled when using BatchScanner optimization"); List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges(); if (ranges.isEmpty()) { ranges = new ArrayList<>(1); TabletLocator tl; try { if (tableConfig.isOfflineScan()) { binnedRanges = binOfflineTable(job, tableId, ranges); while (binnedRanges == null) { org.apache.accumulo.core.clientImpl.mapreduce.SplitUtils.updateSplit(split, tableConfig, logLevel); split.setOffline(tableConfig.isOfflineScan()); split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners()); split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators()); splits.add(split); } else {
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "Usage : " + MRTester.class.getName() + " <table1> <table2>"); } String user = getAdminPrincipal(); AuthenticationToken pass = getAdminToken(); String table1 = args[0]; String table2 = args[1]; JobConf job = new JobConf(getConf()); job.setJarByClass(this.getClass()); job.setInputFormat(AccumuloInputFormat.class); AccumuloMultiTableInputFormat.setConnectorInfo(job, user, pass); AccumuloMultiTableInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig()); InputTableConfig tableConfig1 = new InputTableConfig(); InputTableConfig tableConfig2 = new InputTableConfig(); Map<String,InputTableConfig> configMap = new HashMap<>(); configMap.put(table1, tableConfig1); configMap.put(table2, tableConfig2); AccumuloMultiTableInputFormat.setInputTableConfigs(job, configMap); job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setOutputFormat(NullOutputFormat.class); job.setNumReduceTasks(0); return JobClient.runJob(job).isSuccessful() ? 0 : 1; }
@Override protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) { return getInputTableConfig(context, tableName).getIterators(); } };
/** * Creates a batch scan config object out of a previously serialized batch scan config object. * * @param input * the data input of the serialized batch scan config */ public InputTableConfig(DataInput input) throws IOException { readFields(input); }
String tableName = getInputTableName(implementingClass, conf); if (tableName != null) { InputTableConfig queryConfig = new InputTableConfig(); List<IteratorSetting> itrs = getIterators(implementingClass, conf); if (itrs != null) queryConfig.setIterators(itrs); Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf); if (columns != null) queryConfig.fetchColumns(columns); List<Range> ranges = null; try { queryConfig.setRanges(ranges); queryConfig.setSamplerConfiguration(samplerConfig); queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)) .setUseIsolatedScanners(isIsolated(implementingClass, conf)) .setUseLocalIterators(usesLocalIterators(implementingClass, conf)) .setOfflineScan(isOfflineScan(implementingClass, conf)); return Maps.immutableEntry(tableName, queryConfig);
Boolean isOffline = baseSplit.isOffline(); if (null == isOffline) { isOffline = tableConfig.isOfflineScan(); isIsolated = tableConfig.shouldUseIsolatedScanners(); usesLocalIterators = tableConfig.shouldUseLocalIterators(); columns = tableConfig.getFetchedColumns(); samplerConfig = tableConfig.getSamplerConfiguration(); executionHints = tableConfig.getExecutionHints();
boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators()); if (batchScan && !supportBatchScan) throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators"); boolean autoAdjust = tableConfig.shouldAutoAdjustRanges(); if (batchScan && !autoAdjust) throw new IllegalArgumentException( "AutoAdjustRanges must be enabled when using BatchScanner optimization"); List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges(); if (ranges.isEmpty()) { ranges = new ArrayList<>(1); TabletLocator tl; try { if (tableConfig.isOfflineScan()) { binnedRanges = binOfflineTable(context, tableId, ranges); while (binnedRanges == null) { ke.clip(r), new String[] {location}); SplitUtils.updateSplit(split, tableConfig, logLevel); split.setOffline(tableConfig.isOfflineScan()); split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners()); split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators()); splits.add(split); } else {
if (!tableConfig.shouldUseLocalIterators()) { if (tableConfig.getIterators() != null) { for (IteratorSetting iter : tableConfig.getIterators()) { if (!conn.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))