@Override public Long convert(String value) { return AccumuloConfiguration.getMemoryInBytes(value); } }
@Override public boolean validateOptions(Map<String,String> options) { String maxBufferSizeStr = options.get(MAX_BUFFER_SIZE_OPT); try { AccumuloConfiguration.getMemoryInBytes(maxBufferSizeStr); } catch (Exception e) { throw new IllegalArgumentException( "Failed to parse opt " + MAX_BUFFER_SIZE_OPT + " " + maxBufferSizeStr, e); } return true; }
@Override public String convert(String str) { long size = AccumuloConfiguration.getMemoryInBytes(str); checkArgument(size > 0); return Long.toString(size); } }
@Override public void init(Map<String,String> options) { limit = AccumuloConfiguration.getMemoryInBytes(options.get(SIZE_LIMIT_OPT)); }
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { sourceIter = source; if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT)); } }
@Override public long getDesiredRate() { return getConfiguration().getMemoryInBytes(Property.TSERV_MAJC_THROUGHPUT); } };
@Override public boolean validateOptions(Map<String,String> options) { for (Entry<String,String> option : options.entrySet()) { try { if (option.getKey().equals(AUTH_OPT)) { new Authorizations(option.getValue().getBytes(UTF_8)); } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) { AccumuloConfiguration.getMemoryInBytes(option.getValue()); } } catch (Exception e) { throw new IllegalArgumentException( "Failed to parse opt " + option.getKey() + " " + option.getValue(), e); } } return true; }
public List<RecoveryStatus> getLogSorts() { List<RecoveryStatus> result = new ArrayList<>(); synchronized (currentWork) { for (Entry<String,LogProcessor> entries : currentWork.entrySet()) { RecoveryStatus status = new RecoveryStatus(); status.name = entries.getKey(); try { status.progress = entries.getValue().getBytesCopied() / (0.0 + conf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE)); } catch (IOException ex) { log.warn("Error getting bytes read"); } status.runtime = (int) entries.getValue().getSortTime(); result.add(status); } return result; } } }
public static CryptoModuleParameters fillParamsObjectFromConfiguration( CryptoModuleParameters params, AccumuloConfiguration conf) { // Get all the options from the configuration Map<String,String> cryptoOpts = new HashMap<>( conf.getAllPropertiesWithPrefix(Property.CRYPTO_PREFIX)); cryptoOpts.putAll(conf.getAllPropertiesWithPrefix(Property.INSTANCE_PREFIX)); cryptoOpts.remove(Property.INSTANCE_SECRET.getKey()); cryptoOpts.put(Property.CRYPTO_BLOCK_STREAM_SIZE.getKey(), Integer.toString((int) conf.getMemoryInBytes(Property.CRYPTO_BLOCK_STREAM_SIZE))); return fillParamsObjectFromStringMap(params, cryptoOpts); }
public List<RecoveryStatus> getLogSorts() { List<RecoveryStatus> result = new ArrayList<RecoveryStatus>(); synchronized (currentWork) { for (Entry<String,LogProcessor> entries : currentWork.entrySet()) { RecoveryStatus status = new RecoveryStatus(); status.name = entries.getKey(); try { status.progress = entries.getValue().getBytesCopied() / (0.0 + conf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE)); } catch (IOException ex) { log.warn("Error getting bytes read"); } status.runtime = (int) entries.getValue().getSortTime(); result.add(status); } return result; } } }
private InetSocketAddress startStatsService(String hostname) throws UnknownHostException { Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this)); int port = instance.getConfiguration().getPort(Property.GC_PORT); long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE); try { port = TServerUtils.startTServer(port, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize).port; } catch (Exception ex) { log.fatal(ex, ex); throw new RuntimeException(ex); } return new InetSocketAddress(hostname, port); }
/** * Gets a property of type {@link PropertyType#MEMORY}, interpreting the value properly. * * @param property * property to get * @return property value * @throws IllegalArgumentException * if the property is of the wrong type * @see #getMemoryInBytes(String) */ public long getMemoryInBytes(Property property) { checkType(property, PropertyType.MEMORY); String memString = get(property); return getMemoryInBytes(memString); }
@Override public void init(ServerConfiguration conf) { this.config = conf; maxMemory = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); maxConcurrentMincs = conf.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT); numWaitingMultiplier = TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER; }
@Override public void init(ServerConfiguration conf) { this.config = conf; maxMemory = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); maxConcurrentMincs = conf.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT); numWaitingMultiplier = Constants.TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER; }
MemoryManagementFramework() { tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>()); memUsageReports = new LinkedBlockingQueue<>(); maxMem = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); Runnable r1 = new Runnable() { @Override public void run() { processTabletMemStats(); } }; memoryGuardThread = new Daemon(new LoggingRunnable(log, r1)); memoryGuardThread.setPriority(Thread.NORM_PRIORITY + 1); memoryGuardThread.setName("Accumulo Memory Guard"); Runnable r2 = new Runnable() { @Override public void run() { manageMemory(); } }; minorCompactionInitiatorThread = new Daemon(new LoggingRunnable(log, r2)); minorCompactionInitiatorThread.setName("Accumulo Minor Compaction Initiator"); }
MemoryManagementFramework() { tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>()); memUsageReports = new LinkedBlockingQueue<TabletStateImpl>(); maxMem = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); Runnable r1 = new Runnable() { public void run() { processTabletMemStats(); } }; memoryGuardThread = new Daemon(new LoggingRunnable(log, r1)); memoryGuardThread.setPriority(Thread.NORM_PRIORITY + 1); memoryGuardThread.setName("Accumulo Memory Guard"); Runnable r2 = new Runnable() { public void run() { manageMemory(); } }; minorCompactionInitiatorThread = new Daemon(new LoggingRunnable(log, r2)); minorCompactionInitiatorThread.setName("Accumulo Minor Compaction Initiator"); }
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); scanning = IteratorScope.scan.equals(env.getIteratorScope()); if (scanning) { String auths = options.get(AUTH_OPT); if (auths != null && !auths.isEmpty()) { ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(UTF_8))); visibleCache = new LRUMap(100); } } if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT)); } else { maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; } parsedVisibilitiesCache = new LRUMap(100); }
@Override public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) { UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID); if (us == null) { throw new RuntimeException("No Such SessionID"); } try { KeyExtent keyExtent = new KeyExtent(tkeyExtent); setUpdateTablet(us, keyExtent); if (us.currentTablet != null) { List<Mutation> mutations = us.queuedMutations.get(us.currentTablet); for (TMutation tmutation : tmutations) { Mutation mutation = new ServerMutation(tmutation); mutations.add(mutation); us.queuedMutationSize += mutation.numBytes(); } if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX)) flush(us); } } finally { sessionManager.unreserveSession(us); } }
private HostAndPort startStatsService() throws UnknownHostException { Iface rpcProxy = RpcWrapper.service(this, new Processor<Iface>(this)); final Processor<Iface> processor; if (ThriftServerType.SASL == getThriftServerType()) { Iface tcProxy = TCredentialsUpdatingWrapper.service(rpcProxy, getClass(), getConfiguration()); processor = new Processor<>(tcProxy); } else { processor = new Processor<>(rpcProxy); } int port[] = getConfiguration().getPort(Property.GC_PORT); HostAndPort[] addresses = TServerUtils.getHostAndPorts(this.opts.getAddress(), port); long maxMessageSize = getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE); try { ServerAddress server = TServerUtils.startTServer(getConfiguration(), getThriftServerType(), processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, getConfiguration().getCount(Property.GENERAL_SIMPLETIMER_THREADPOOL_SIZE), 1000, maxMessageSize, getServerSslParams(), getSaslParams(), 0, addresses); log.debug("Starting garbage collector listening on " + server.address); return server.address; } catch (Exception ex) { // ACCUMULO-3651 Level changed to error and FATAL added to message for slf4j compatibility log.error("FATAL:", ex); throw new RuntimeException(ex); } }
public void start(String[] args) throws MergeException { Opts opts = new Opts(); opts.parseArgs(Merge.class.getName(), args); try { Connector conn = opts.getConnector(); if (!conn.tableOperations().exists(opts.getTableName())) { System.err.println("table " + opts.getTableName() + " does not exist"); return; } if (opts.goalSize == null || opts.goalSize < 1) { AccumuloConfiguration tableConfig = new ConfigurationCopy( conn.tableOperations().getProperties(opts.getTableName())); opts.goalSize = tableConfig.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD); } message("Merging tablets in table %s to %d bytes", opts.getTableName(), opts.goalSize); mergomatic(conn, opts.getTableName(), opts.begin, opts.end, opts.goalSize, opts.force); } catch (Exception ex) { throw new MergeException(ex); } }