/** * Given options for tables (across multiple volumes), construct an absolute path using the unique * name within the chosen volume * * @return An absolute, unique path for the imported table */ protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) { // We can try to spread out the tablet dirs across all volumes VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableInfo.tableId, master.getContext()); String tableDir = master.getFileSystem().choose(chooserEnv, tableDirs); // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX return tableDir + "/" + tableInfo.tableId + "/" + tabletDir; }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj == null || !(obj instanceof VolumeChooserEnvironment)) { return false; } VolumeChooserEnvironment other = (VolumeChooserEnvironment) obj; return getScope() == other.getScope() && Objects.equals(getTableId(), other.getTableId()); }
ServerConfigurationFactory loadConfFactory(VolumeChooserEnvironment env) { // Get the current table's properties, and find the preferred volumes property // This local variable is an intentional component of the single-check idiom. ServerConfigurationFactory localConf = lazyConfFactory; if (localConf == null) { // If we're under contention when first getting here we'll throw away some initializations. localConf = env.getServerContext().getServerConfFactory(); lazyConfFactory = localConf; } return localConf; } }
@Override public String choose(VolumeChooserEnvironment env, String[] options) { VolumeChooser chooser = null; if (env.hasTableId()) { serverConfs = localConf; final TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId()); chooser = tableSpecificChooser.get(env.getTableId()); if (chooser == null) { VolumeChooser temp = Property.createTableInstanceFromPropertyName(tableConf, Property.TABLE_VOLUME_CHOOSER, VolumeChooser.class, fallbackVolumeChooser); chooser = tableSpecificChooser.putIfAbsent(env.getTableId(), temp); if (chooser == null) { chooser = temp; VolumeChooser last = tableSpecificChooser.replace(env.getTableId(), temp); if (chooser.equals(last)) { chooser = temp;
VolumeChooser getDelegateChooser(VolumeChooserEnvironment env) { switch (env.getScope()) { case INIT: // TODO should be possible to read from SiteConfiguration during init log.warn("Not possible to determine delegate chooser at '{}' scope. Using {}.", ChooserScope.INIT, RandomVolumeChooser.class.getName()); return randomChooser; case TABLE: return getVolumeChooserForTable(env, loadConfFactory(env)); default: return getVolumeChooserForScope(env, loadConfFactory(env)); } }
private VolumeChooser getVolumeChooserForTable(VolumeChooserEnvironment env, ServerConfigurationFactory confFactory) { log.trace("Looking up property {} for table id: {}", TABLE_VOLUME_CHOOSER, env.getTableId()); final TableConfiguration tableConf = confFactory.getTableConfiguration(env.getTableId()); String clazz = tableConf.get(TABLE_VOLUME_CHOOSER); // fall back to global default scope, so setting only one default is necessary, rather than a // separate default for TABLE scope than other scopes if (clazz == null || clazz.isEmpty()) { clazz = confFactory.getSystemConfiguration().get(DEFAULT_SCOPED_VOLUME_CHOOSER); } if (clazz == null || clazz.isEmpty()) { String msg = "Property " + TABLE_VOLUME_CHOOSER + " or " + DEFAULT_SCOPED_VOLUME_CHOOSER + " must be a valid " + VolumeChooser.class.getSimpleName() + " to use the " + getClass().getSimpleName(); throw new VolumeChooserException(msg); } String context = getTableContext(tableConf); // can be null return createVolumeChooser(context, clazz, TABLE_VOLUME_CHOOSER, env.getTableId(), tableSpecificChooserCache); }
@Override public String choose(VolumeChooserEnvironment env, String[] options) { if (!env.hasTableId()) return super.choose(env, options); serverConfs = localConf; TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId()); final Map<String,String> props = new HashMap<>(); tableConf.getProperties(props, PREFERRED_VOLUMES_FILTER); log.trace("In custom chooser"); log.trace("Volumes: " + volumes); log.trace("TableID: " + env.getTableId());
String[] getPreferredVolumes(VolumeChooserEnvironment env, String[] options) { switch (env.getScope()) { case INIT: // TODO should be possible to read from SiteConfiguration during init log.warn("Not possible to determine preferred volumes at '{}' scope. Using all volumes.", ChooserScope.INIT); return options; case TABLE: return getPreferredVolumesForTable(env, loadConfFactory(env), options); default: return getPreferredVolumesForScope(env, loadConfFactory(env), options); } }
private String[] getPreferredVolumesForTable(VolumeChooserEnvironment env, ServerConfigurationFactory confFactory, String[] options) { log.trace("Looking up property {} + for Table id: {}", TABLE_PREFERRED_VOLUMES, env.getTableId()); final TableConfiguration tableConf = confFactory.getTableConfiguration(env.getTableId()); String preferredVolumes = tableConf.get(TABLE_PREFERRED_VOLUMES); // fall back to global default scope, so setting only one default is necessary, rather than a // separate default for TABLE scope than other scopes if (preferredVolumes == null || preferredVolumes.isEmpty()) { preferredVolumes = confFactory.getSystemConfiguration().get(DEFAULT_SCOPED_PREFERRED_VOLUMES); } // throw an error if volumes not specified or empty if (preferredVolumes == null || preferredVolumes.isEmpty()) { String msg = "Property " + TABLE_PREFERRED_VOLUMES + " or " + DEFAULT_SCOPED_PREFERRED_VOLUMES + " must be a subset of " + Arrays.toString(options) + " to use the " + getClass().getSimpleName(); throw new VolumeChooserException(msg); } return parsePreferred(TABLE_PREFERRED_VOLUMES, preferredVolumes, options); }
Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet"); if (fs.exists(oldPath)) { VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(RootTable.ID, context); String newPath = fs.choose(chooserEnv, ServerConstants.getBaseUris(context)) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
private VolumeChooser getVolumeChooserForScope(VolumeChooserEnvironment env, ServerConfigurationFactory confFactory) { ChooserScope scope = env.getScope(); String property = getPropertyNameForScope(scope); log.trace("Looking up property {} for scope: {}", property, scope); AccumuloConfiguration systemConfiguration = confFactory.getSystemConfiguration(); String clazz = systemConfiguration.get(property); // fall back to global default scope if this scope isn't configured (and not already default // scope) if ((clazz == null || clazz.isEmpty()) && scope != ChooserScope.DEFAULT) { log.debug("{} not found; using {}", property, DEFAULT_SCOPED_VOLUME_CHOOSER); clazz = systemConfiguration.get(DEFAULT_SCOPED_VOLUME_CHOOSER); if (clazz == null || clazz.isEmpty()) { String msg = "Property " + property + " or " + DEFAULT_SCOPED_VOLUME_CHOOSER + " must be a valid " + VolumeChooser.class.getSimpleName() + " to use the " + getClass().getSimpleName(); throw new VolumeChooserException(msg); } property = DEFAULT_SCOPED_VOLUME_CHOOSER; } String context = null; return createVolumeChooser(context, clazz, property, scope, scopeSpecificChooserCache); }
ServerConfigurationFactory loadConfFactory(VolumeChooserEnvironment env) { // This local variable is an intentional component of the single-check idiom. ServerConfigurationFactory localConf = lazyConfFactory; if (localConf == null) { // If we're under contention when first getting here we'll throw away some initializations. localConf = env.getServerContext().getServerConfFactory(); lazyConfFactory = localConf; } return localConf; }
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(extent.getTableId(), context); Path newDir = new Path(vm.choose(chooserEnv, ServerConstants.getBaseUris(context))
private String[] getPreferredVolumesForScope(VolumeChooserEnvironment env, ServerConfigurationFactory confFactory, String[] options) { ChooserScope scope = env.getScope(); String property = getPropertyNameForScope(scope); log.trace("Looking up property {} for scope: {}", property, scope); AccumuloConfiguration systemConfiguration = confFactory.getSystemConfiguration(); String preferredVolumes = systemConfiguration.get(property); // fall back to global default scope if this scope isn't configured (and not already default // scope) if ((preferredVolumes == null || preferredVolumes.isEmpty()) && scope != ChooserScope.DEFAULT) { log.debug("{} not found; using {}", property, DEFAULT_SCOPED_PREFERRED_VOLUMES); preferredVolumes = systemConfiguration.get(DEFAULT_SCOPED_PREFERRED_VOLUMES); // only if the custom property is not set to we fall back to the default scoped preferred // volumes if (preferredVolumes == null || preferredVolumes.isEmpty()) { String msg = "Property " + property + " or " + DEFAULT_SCOPED_PREFERRED_VOLUMES + " must be a subset of " + Arrays.toString(options) + " to use the " + getClass().getSimpleName(); throw new VolumeChooserException(msg); } property = DEFAULT_SCOPED_PREFERRED_VOLUMES; } return parsePreferred(property, preferredVolumes, options); }
public WeightedRandomCollection(List<String> options, VolumeChooserEnvironment env, Random random) { this.random = random; if (options.size() < 1) { throw new IllegalStateException("Options was empty! No valid volumes to choose from."); } VolumeManager manager = env.getServerContext().getVolumeManager(); // Compute percentage space available on each volume for (String option : options) { FileSystem pathFs = manager.getVolumeByPath(new Path(option)).getFileSystem(); try { FsStatus optionStatus = pathFs.getStatus(); double percentFree = ((double) optionStatus.getRemaining() / optionStatus.getCapacity()); add(percentFree, option); } catch (IOException e) { log.error("Unable to get file system status for" + option, e); } } if (map.size() < 1) { throw new IllegalStateException( "Weighted options was empty! Could indicate an issue getting file system status or " + "no free space on any volume"); } }
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context); String volume = fs.choose(chooserEnv, ServerConstants.getBaseUris(context)) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR;
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ChooserScope.INIT, null); final String rootTabletDir = new Path( fs.choose(chooserEnv, configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR
VolumeManager fs = conf.getFileSystem(); VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ChooserScope.LOGGER, context); logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris(context)) + Path.SEPARATOR
Mutation m = new Mutation(key.getRow()); VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId, context); final String newLocation = vm.choose(chooserEnv, ServerConstants.getBaseUris(context)) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
/** * During an upgrade from 1.6 to 1.7, we need to add the replication table */ public static void createReplicationTable(ServerContext context) { VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ReplicationTable.ID, context); String dir = context.getVolumeManager().choose(chooserEnv, ServerConstants.getBaseUris(context)) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID + Constants.DEFAULT_TABLET_LOCATION; Mutation m = new Mutation(new Text(TabletsSection.getRow(ReplicationTable.ID, null))); m.put(DIRECTORY_COLUMN.getColumnFamily(), DIRECTORY_COLUMN.getColumnQualifier(), 0, new Value(dir.getBytes(UTF_8))); m.put(TIME_COLUMN.getColumnFamily(), TIME_COLUMN.getColumnQualifier(), 0, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); m.put(PREV_ROW_COLUMN.getColumnFamily(), PREV_ROW_COLUMN.getColumnQualifier(), 0, KeyExtent.encodePrevEndRow(null)); update(context, getMetadataTable(context), null, m); }