protected DomainData getDomainData(Domain domain) throws IOException { DomainData data = new DomainData(); data.name = domain.getName(); data.numPartitions = domain.getNumParts(); data.storageEngine = domain.getStorageEngineFactoryClassName(); data.storageEngineOptions = domain.getStorageEngineOptions(); Map<Integer, DomainVersionData> versionsMap = new HashMap<Integer, DomainVersionData>(); for (DomainVersion v : domain.getVersions()) { versionsMap.put(v.getVersionNumber(), getDomainVersionData(v)); } data.versionsMap = versionsMap; return data; }
@Override public DomainAndVersion getDomainVersion(Domain domain) throws IOException { if (domain == null || domain.getName() == null) { return null; } for (DomainAndVersion domainVersion : getDomainVersions()) { if (domainVersion.getDomain() != null && domain.getId() == domainVersion.getDomain().getId()) { return domainVersion; } } return null; }
@Override public void run(Coordinator coordinator) throws IOException { Domain domain = DomainBuilderProperties.getDomain(coordinator, domainName); storageEngine = domain.getStorageEngine(); partitioner = domain.getPartitioner(); numPartitions = domain.getNumParts(); } });
public static DomainVersion getDomainVersion(Coordinator coordinator, String domainName, Integer domainVersionNumber) throws IOException { if (domainVersionNumber == null) { return null; } Domain domain = getDomain(coordinator, domainName); DomainVersion domainVersion = domain.getVersionShallow(domainVersionNumber); if (domainVersion == null) { System.out.println(domain.getVersions()); throw new IOException("Could not get version " + domainVersionNumber + " of domain " + domainName + " with coordinator: " + coordinator); } else { return domainVersion; } }
try { int partition = domain.getPartitioner().partition(key, domain.getNumParts()); int keyHash = domain.getPartitioner().partition(key, Integer.MAX_VALUE); partitionToConnectionPool = domainToPartitionToConnectionPool.get(domain.getId()); LOG.error(getLogPrefix() + String.format("Could not find domain to partition map for domain %s (id: %d)", domain.getName(), domain.getId())); return NO_REPLICA; if (hostConnectionPool == null) { LOG.error(getLogPrefix() + String.format("Could not find list of hosts for domain %s (id: %d) when looking for partition %d", domain.getName(), domain.getId(), partition)); return NO_REPLICA; LOG.trace("Looking in domain " + domain.getName() + ", in partition " + partition + ", for key: " + BytesUtils.bytesToHexString(key)); LOG.error(getLogPrefix() + "Failed to perform get: domain " + domain.getName() + ", partition " + partition + ", key: " + BytesUtils.bytesToHexString(key) + ", partitioner: " + domain.getPartitioner() + ", response: " + response);
int numParts = domain.getNumParts(); List<String> requiredFlags = domain.getRequiredHostFlags(); String storageEngineName = domain.getStorageEngineFactoryClassName(); String partitionerName = domain.getPartitionerClassName(); Map<String, Object> storageEngineOptions = domain.getStorageEngineOptions(); LOG.info("Updating domain with new configurations."); coordinator.updateDomain( domain.getName(), configuredParts, configuredStorageEngine,
private int getMaxPartitionsPerHost(Domain domain, int numHosts) { if (domain.getNumParts() % numHosts == 0) { return domain.getNumParts() / numHosts; } else { return (domain.getNumParts() / numHosts) + 1; } } }
domain.getVersion(targetDomainAndVersion.getVersionNumber()); LOG.info(String.format( "Skipping partition update of domain %s partition %d to version %d (it is already up-to-date).", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber())); return; StorageEngine storageEngine = domain.getStorageEngine(); LOG.info(String.format( "Starting partition update of domain %s partition %d to version %d in %s.", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber(), getDataDirectory())); storageEngine.getUpdater(assignment, partition.getPartitionNumber()).updateTo(targetDomainVersion, statistics); LOG.info(String.format( "Completed partition update of domain %s partition %d to version %d.", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber())); domain.getName(), partition.getPartitionNumber()), t); encounteredThrowables.add(t); } finally {
LOG.info("Deleting remote old versions of Domain " + domain.getName() + " (keeping " + numRemoteLeafVersionsToKeep + " newest leaf versions)"); for (DomainVersion version : domain.getVersions()) { if (DomainVersions.isClosed(version)) { sortedVersions.add(version); LOG.info("Keeping Version " + version.getVersionNumber() + " of Domain " + domain.getName()); LOG.info("Deleting Version " + version.getVersionNumber() + " of Domain " + domain.getName()); domain.deleteVersion(version.getVersionNumber());
public static void cleanDomains(Collection <Domain> domains, boolean deleteMetadata, StorageEngine.RemoteLocation dataLocation) throws IOException { for (Domain domain : domains) { StorageEngine storageEngine = domain.getStorageEngine(); RemoteDomainCleaner cleaner = storageEngine.getRemoteDomainCleaner(); if (cleaner == null) { LOG.info("Failed to clean Domain " + domain.getName() + ". No Remote Domain Cleaner is configured."); continue; } RemoteDomainVersionDeleter deleter = storageEngine.getRemoteDomainVersionDeleter(dataLocation); if (deleter == null) { LOG.info("Failed to clean Domain " + domain.getName() + ". No Remote Domain Version Deleter is configured."); continue; } LOG.info("Cleaning Domain " + domain.getName()); cleaner.deleteOldVersions(deleter, deleteMetadata); } }
public static DomainVersion getParentDomainVersion(Domain domain, DomainVersion version) throws IOException { IncrementalDomainVersionProperties properties = (IncrementalDomainVersionProperties)version.getProperties(); if (properties == null) { throw new IOException("Failed to get parent of Domain Version since corresponding properties are empty: " + version); } else { Integer parentVersionNumber = properties.getParentVersionNumber(); if (parentVersionNumber == null) { return null; } else { DomainVersion result = domain.getVersion(parentVersionNumber); if (result == null) { throw new IOException("Failed to get parent Domain Version since specified parent version number (" + parentVersionNumber + ") of version " + version.getVersionNumber() + " of Domain " + domain.getName() + " does not correspond to any version."); } return result; } } }
private Set<DomainVersion> detectCachedVersions(SortedSet<CueballFilePath> cachedFiles) throws IOException { Set<DomainVersion> cachedVersions = new HashSet<DomainVersion>(); for (CueballFilePath file : cachedFiles) { DomainVersion version = domain.getVersion(file.getVersion()); if (version != null) { cachedVersions.add(version); } } return cachedVersions; }
@Override public Domain getDomainById(int domainId) { for (Domain domain : getDomains()) { if (domain.getId() == domainId) { return domain; } } return null; }
@Override public void run(Coordinator coordinator) throws IOException { Domain domain = getDomain(coordinator, domainName); DomainVersion domainVersion = domain.openNewVersion(domainVersionProperties); if (domainVersion == null) { throw new IOException("Could not open a new version of domain " + domainName); } else { LOG.info("Opened new version #" + domainVersion.getVersionNumber() + " of domain: " + domainName); result = new DomainVersionNumberAndNumPartitions(domainVersion.getVersionNumber(), domain.getNumParts()); } } }
@Override public InputSplit[] getSplits(final JobConf conf, int ignored) throws IOException { final String domainName = DomainBuilderProperties.getDomainName(conf); RunWithCoordinator.run(DomainBuilderProperties.getConfigurator(conf), new RunnableWithCoordinator() { @Override public void run(Coordinator coordinator) throws IOException { domain = DomainBuilderProperties.getDomain(coordinator, domainName); domainVersionToCompact = domain.getVersion(DomainCompactorProperties.getVersionNumberToCompact(domainName, conf)); } }); final int domainNumParts = domain.getNumParts(); final StorageEngine storageEngine = domain.getStorageEngine(); final InputSplit[] splits = new InputSplit[domainNumParts]; // Create splits for (int partition = 0; partition < domainNumParts; ++partition) { // Compute remote partition file paths for this split if possible String[] locations = new String[]{}; if (storageEngine instanceof IncrementalStorageEngine) { IncrementalUpdatePlanner updatePlanner = ((IncrementalStorageEngine)storageEngine).getUpdatePlanner(domain); IncrementalUpdatePlan updatePlan = updatePlanner.computeUpdatePlan(domainVersionToCompact); List<String> paths = updatePlanner.getRemotePartitionFilePaths(updatePlan, storageEngine.getPartitionRemoteFileOps(StorageEngine.RemoteLocation.DOMAIN_BUILDER, partition)); locations = LocalityHelper.getHostsSortedByLocality(paths, conf); } splits[partition] = new HadoopDomainCompactorInputSplit(domainName, partition, locations); } return splits; }
if (host.getFlags().containsAll(domain.getRequiredHostFlags()) || host.getFlags().contains(Hosts.ALL_FLAGS_EXPRESSION)) { validHosts.add(new HostAndIndexInRing(host, hostIndex)); LOG.error("Unable to assign Domain " + domain.getName() + " to Ring " + ring.toString() + " since no Host in the Ring is valid for: " + domain.getName());
public static void main(String[] args) throws IOException, InvalidConfigurationException { CommandLineChecker.check(args, new String[]{"configuration", "domain name", "domain version number"}, RemoteDomainVersionDeletionHelper.class ); String configurationPath = args[0]; String domainName = args[1]; Integer versionNumber = Integer.parseInt(args[2]); Coordinator coordinator = new YamlCoordinatorConfigurator(configurationPath).createCoordinator(); Domain domain = coordinator.getDomain(domainName); if (domain == null) { throw new RuntimeException("Given domain was not found: " + domainName); } DomainVersion domainVersion = domain.getVersion(versionNumber); if (domainVersion == null) { throw new RuntimeException("Given version was not found: " + domainName + " version " + versionNumber); } LOG.info("Deleting remote data for domain " + domainName + " version " + versionNumber); domainVersion.setDefunct(true); domain.getStorageEngine().getRemoteDomainVersionDeleter(StorageEngine.RemoteLocation.DOMAIN_BUILDER).deleteVersion(versionNumber); } }
public static DomainVersion getLatestVersion(Domain domain) throws IOException { SortedSet<DomainVersion> versions = domain.getVersions(); if (versions == null || versions.size() == 0) { return null; } else { return versions.last(); } }
@Override protected void action(HttpServletRequest req, HttpServletResponse resp) throws IOException { Domain domain = DomainController.this.coordinator.getDomain(req.getParameter("n")); for (DomainVersion domainVersion : domain.getVersions()) { if (domainVersion.isDefunct()) { domain.deleteVersion(domainVersion.getVersionNumber()); } } redirect("/domain.jsp?n=" + req.getParameter("n"), resp); } });