@Override public AccumuloConfiguration getConfig() { return AccumuloConfiguration.getDefaultConfiguration(); }
public BulkIteratorEnvironment(IteratorScope scope) { this.scope = scope; this.conf = AccumuloConfiguration.getDefaultConfiguration(); }
public RFileEnvironment() { this.conf = AccumuloConfiguration.getDefaultConfiguration(); }
public MockNamespace() { settings = new HashMap<>(); for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) { settings.put(key, entry.getValue()); } } }
public Writer(BlockFileWriter bfw, int blockSize) throws IOException { this(bfw, blockSize, (int) AccumuloConfiguration.getDefaultConfiguration() .getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX), null, null); }
@Override public AccumuloConfiguration getSiteConfiguration() { Configuration conf = new Configuration(false); Path accumuloSite = new Path(serverAccumuloConfDir, "accumulo-site.xml"); conf.addResource(accumuloSite); return new ConfigurationCopy( Iterables.concat(AccumuloConfiguration.getDefaultConfiguration(), conf)); } }
public MockIteratorEnvironment(boolean major) { this.conf = AccumuloConfiguration.getDefaultConfiguration(); this.major = major; }
public MockIteratorEnvironment() { this.conf = AccumuloConfiguration.getDefaultConfiguration(); }
MockTable(boolean limitVersion, TimeType timeType, String tableId) { this.timeType = timeType; this.tableId = tableId; settings = IteratorUtil.generateInitialTableProperties(limitVersion); for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); } }
@Override public AccumuloConfiguration getSiteConfiguration() { // TODO Auto-generated method stub return new ConfigurationCopy(Iterables.concat(AccumuloConfiguration.getDefaultConfiguration(), config.getSiteConfig().entrySet())); } }
public MockTable(MockNamespace namespace, TimeType timeType, String tableId, Map<String,String> properties) { this.timeType = timeType; this.tableId = tableId; settings = properties; for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); } Set<Entry<String,String>> set = namespace.settings.entrySet(); Iterator<Entry<String,String>> entries = set.iterator(); while (entries.hasNext()) { Entry<String,String> entry = entries.next(); String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); } }
public RFileBatchWriter(Configuration conf, FileSystem fs, String file) throws IOException { AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); CachableBlockFile.Writer cbw = new CachableBlockFile.Writer(PositionedOutputs .wrap(fs.create(new Path(file), false, conf.getInt("io.file.buffer.size", 4096), (short) conf.getInt("dfs.replication", 3), conf.getLong("dfs.block.size", 1 << 26))), "gz", conf, aconf); writer = new RFile.Writer(cbw, 100000); writer.startDefaultLocalityGroup(); }
private ThreadPoolExecutor createExecutorService(final String prop, final String name) { final ThreadPoolExecutor service = createExecutorService(getMaxThreads(prop), name + " (" + instanceId + ')'); threadPools.put(name, service); SimpleTimer.getInstance(AccumuloConfiguration.getDefaultConfiguration()).schedule(() -> { try { int max = getMaxThreads(prop); if (service.getMaximumPoolSize() != max) { log.info("Changing " + prop + " to " + max); service.setCorePoolSize(max); service.setMaximumPoolSize(max); } } catch (Throwable t) { log.error(t, t); } }, 1000, 10 * 1000); return service; }
@Override public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt) throws IOException, InterruptedException { // grab the configuration final Configuration conf = attempt.getConfiguration(); // create a filename final String filenamePrefix = getPathName(conf); final String taskID = attempt.getTaskAttemptID().toString(); // grab the max size final long maxSize = getMaxBufferSize(conf); // grab the FileSystem final FileSystem fs = FileSystem.get(conf); // create a default AccumuloConfiguration final AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); return new BufferingRFileRecordWriter(maxSize, acuconf, conf, filenamePrefix, taskID, fs); }
@Override public Pair<String,String> call() throws Exception { Path bulkFailures = new Path(base, "failures" + which); Path files = new Path(base, "files" + which); fs.mkdirs(bulkFailures); fs.mkdirs(files); for (int i = 0; i < 100; i++) { FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder() .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); writer.startDefaultLocalityGroup(); for (int j = 0x100; j < 0xfff; j += 3) { writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0])); } writer.close(); } return new Pair<>(files.toString(), bulkFailures.toString()); } }));
private static void createMetadataFile(VolumeManager volmanager, String fileName, Tablet... tablets) throws IOException { // sort file contents in memory, then play back to the file TreeMap<Key,Value> sorted = new TreeMap<>(); for (Tablet tablet : tablets) { createEntriesForTablet(sorted, tablet); } FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem(); FileSKVWriter tabletWriter = FileOperations.getInstance().newWriterBuilder() .forFile(fileName, fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); tabletWriter.startDefaultLocalityGroup(); for (Entry<Key,Value> entry : sorted.entrySet()) { tabletWriter.append(entry.getKey(), entry.getValue()); } tabletWriter.close(); }
@Override public Map<String,String> getConfiguration(TInfo tinfo, TCredentials credentials, ConfigurationType type) throws TException { switch (type) { case CURRENT: return conf(credentials, new ServerConfiguration(instance).getConfiguration()); case SITE: return conf(credentials, ServerConfiguration.getSiteConfiguration()); case DEFAULT: return conf(credentials, AccumuloConfiguration.getDefaultConfiguration()); } throw new RuntimeException("Unexpected configuration type " + type); }
@Override public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); return RFileOperations.getInstance().newReaderBuilder().forFile(mapFileName, fs, conf) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { FileSplit fileSplit = (FileSplit) split; // Note that the RFileInputFormat returns false for "isSplittable", so this should ALWAYS be 0 start = fileSplit.getStart(); if (start != 0) { throw new IOException("Cannot start reading an RFile in the middle: start=" + start); } end = fileSplit.getLength() - start; pos = start; FileOperations ops = RFileOperations.getInstance(); String file = fileSplit.getPath().toString(); FileSystem fs = fileSplit.getPath().getFileSystem(context.getConfiguration()); fileIterator = ops.newReaderBuilder().forFile(file, fs, context.getConfiguration()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning().build(); }
@Override public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); return RFileOperations.getInstance().newReaderBuilder().forFile(mapFileName, fs, conf) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning().build(); }