@Override public int compareTo(PartitionUpdateTask other) { if (partition.getPartitionNumber() < other.partition.getPartitionNumber()) { return -1; } else if (partition.getPartitionNumber() > other.partition.getPartitionNumber()) { return 1; } else { return 0; } } }
if (partitions != null) { for (HostDomainPartition partition : partitions) { if (partition.getCurrentDomainVersion() != null) { numTotalPartitions += 1; partitionNumbers.add(partition.getPartitionNumber()); if (partition.getCurrentDomainVersion() == null) { LOG.error(String.format( "Could not load Reader for partition #%d of Domain %s because the partition's current version is null.", partition.getPartitionNumber(), domain.getName())); continue; reader = engine.getReader(configurator.getReaderConfigurator(numTotalPartitions), partition.getPartitionNumber(), assignment); } catch (IOException | IllegalArgumentException e) { if (!partition.isDeletable()) { partition.setDeletable(true); partition.getPartitionNumber(), domain.getName()); LOG.error(msg, e); exceptions.add(new IOException(msg, e)); if (reader.getVersionNumber() != null && !reader.getVersionNumber().equals(partition.getCurrentDomainVersion())) { partition.setDeletable(true); final String msg = String.format("Could not load Reader for partition #%d of domain %s because version numbers reported by the Reader (%d) and by metadata (%d) differ.", partition.getPartitionNumber(), domain.getName(), reader.getVersionNumber(), partition.getCurrentDomainVersion()); LOG.error(msg);
if (!partition.isDeletable() && (partitionMappings == null || !partitionMappings.contains(partition.getPartitionNumber()))) { partition.setDeletable(true); if (!domains.contains(hostDomain.getDomain())) { for (HostDomainPartition partition : hostDomain.getPartitions()) { if (!partition.isDeletable()) { partition.setDeletable(true);
if (partition.isDeletable() || targetDomainAndVersion == null) { deletePartition(hostDomain, partition); } else { if (partition.getCurrentDomainVersion() != null && partition.getCurrentDomainVersion().equals(targetDomainVersion.getVersionNumber())) { LOG.info(String.format( "Skipping partition update of domain %s partition %d to version %d (it is already up-to-date).", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber())); return; partition.setCurrentDomainVersion(null); LOG.info(String.format( "Starting partition update of domain %s partition %d to version %d in %s.", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber(), getDataDirectory())); storageEngine.getUpdater(assignment, partition.getPartitionNumber()).updateTo(targetDomainVersion, statistics); partition.setCurrentDomainVersion(targetDomainVersion.getVersionNumber()); LOG.info(String.format( "Completed partition update of domain %s partition %d to version %d.", domain.getName(), partition.getPartitionNumber(), targetDomainVersion.getVersionNumber())); domain.getName(), partition.getPartitionNumber()), t); encounteredThrowables.add(t); } finally {
public static Map<Domain, Map<Integer, Set<Host>>> domainToPartitionToHostsServing(RingGroup ringGroup, HostReplicaStatus config) throws IOException { Map<Domain, Map<Integer, Set<Host>>> result = new HashMap<>(); // Compute num replicas fully serving for all partitions for (Ring ring : ringGroup.getRings()) { for (Host h : ring.getHosts()) { if (isFullyServing(h, false, config)) { for (HostDomain hostDomain : h.getAssignedDomains()) { Domain domain = hostDomain.getDomain(); for (HostDomainPartition partition : hostDomain.getPartitions()) { if (!partition.isDeletable() && partition.getCurrentDomainVersion() != null) { int partitionNumber = partition.getPartitionNumber(); Map<Integer, Set<Host>> partitionToNumFullyServing = result.get(domain); if (partitionToNumFullyServing == null) { partitionToNumFullyServing = new HashMap<Integer, Set<Host>>(); result.put(domain, partitionToNumFullyServing); } if (!partitionToNumFullyServing.containsKey(partitionNumber)) { partitionToNumFullyServing.put(partitionNumber, new HashSet<Host>()); } partitionToNumFullyServing.get(partitionNumber).add(h); } } } } } } return result; }
if (!partition.isDeletable() && (partitionMappings == null || !partitionMappings.contains(partition.getPartitionNumber()))) { return false; if (!domains.contains(hostDomain.getDomain())) { for (HostDomainPartition partition : hostDomain.getPartitions()) { if (!partition.isDeletable()) { return false;
public static UpdateProgressAggregator computeUpdateProgress(Host host, DomainGroup domainGroup) throws IOException { UpdateProgressAggregator result = new UpdateProgressAggregator(); for (DomainAndVersion dgvdv : domainGroup.getDomainVersions()) { Domain domain = dgvdv.getDomain(); HostDomain hostDomain = host.getHostDomain(domain); if (hostDomain != null) { for (HostDomainPartition partition : hostDomain.getPartitions()) { // Ignore deletable partitions if (!partition.isDeletable()) { if (partition.getCurrentDomainVersion() != null && partition.getCurrentDomainVersion().equals(dgvdv.getVersionNumber())) { result.add(domain, true); } else { result.add(domain, false); } } } } } return result; }
Integer versionNumber = hostDomainPartition.getCurrentDomainVersion(); if (versionNumber != null) { for (String filePath : storageEngine.getFiles(assignments, versionNumber, hostDomainPartition.getPartitionNumber())) { File file = new File(filePath);
private void doDeleteOrUndeletePartition(HttpServletRequest req, HttpServletResponse resp, boolean deletable) throws IOException { RingGroup rg = coordinator.getRingGroup(req.getParameter("g")); Ring r = rg.getRing(Integer.parseInt(req.getParameter("n"))); Host h = r.getHostByAddress(PartitionServerAddress.parse(URLEnc.decode(req.getParameter("h")))); HostDomain hd = h.getHostDomain(coordinator.getDomain(req.getParameter("d"))); HostDomainPartition hdp = hd.getPartitionByNumber(Integer.parseInt(req.getParameter("p"))); hdp.setDeletable(deletable); redirectBack(resp, rg, r, h); }
private static boolean isAssignedDeletablePartition(Host host) throws IOException { for (HostDomain hostDomain : host.getAssignedDomains()) { for (HostDomainPartition partition : hostDomain.getPartitions()) { if (partition.isDeletable()) { return true; } } } return false; }
public static boolean isServable(Host host) throws IOException { int numPartitions = 0; for (HostDomain hostDomain : host.getAssignedDomains()) { for (HostDomainPartition hostDomainPartition : hostDomain.getPartitions()) { ++numPartitions; if (hostDomainPartition.getCurrentDomainVersion() == null) { return false; } } } return numPartitions != 0; }
public static ServingStatusAggregator computeServingStatusAggregator(Host host, DomainGroup domainGroup) throws IOException { ServingStatusAggregator result = new ServingStatusAggregator(); for (HostDomain hostDomain : host.getAssignedDomains()) { DomainAndVersion domainVersion = domainGroup.getDomainVersion(hostDomain.getDomain()); // Ignore domains that are not relevant if (domainVersion != null) { for (HostDomainPartition partition : hostDomain.getPartitions()) { // Ignore deletable partitions if (!partition.isDeletable()) { // Check if partition is served and up to date boolean servedAndUpToDate = host.getState() == HostState.SERVING && partition.getCurrentDomainVersion() != null && partition.getCurrentDomainVersion().equals(domainVersion.getVersionNumber()); // Aggregate counts result.add(hostDomain.getDomain(), partition.getPartitionNumber(), servedAndUpToDate); } } } } return result; }
if (!partition.isDeletable()) { List<HostAddress> partitionsList = partitionToAdresses.get(partition.getPartitionNumber()); if (partitionsList == null) { partitionsList = new ArrayList<HostAddress>(); partitionToAdresses.put(partition.getPartitionNumber(), partitionsList);
private static boolean allPartitionsUpToDate(Host host, Collection<DomainAndVersion> domainVersions, boolean allowMoreRecentDomainVersions) throws IOException { // Check that each domain of the given domain group version is up to date on this host for (DomainAndVersion domainAndVersion : domainVersions) { Domain domain = domainAndVersion.getDomain(); HostDomain hostDomain = host.getHostDomain(domain); if (hostDomain != null) { for (HostDomainPartition partition : hostDomain.getPartitions()) { // Ignore deletable partitions if (!partition.isDeletable()) { // If the partition is not currently at the given domain group version, the host is not up-to-date if (partition.getCurrentDomainVersion() == null || (!allowMoreRecentDomainVersions && partition.getCurrentDomainVersion() != domainAndVersion.getVersionNumber()) || (allowMoreRecentDomainVersions && (partition.getCurrentDomainVersion() < domainAndVersion.getVersionNumber()))) { LOG.info("Host " + host.getAddress().getHostName() + " is not up to date for domain " + domain.getName() + " and version " + domainAndVersion.getVersionNumber()); LOG.info("Host " + host.getAddress().getHostName() + " has a partition on domain " + domain.getName() + " at version " + partition.getCurrentDomainVersion()); return false; } } } } } return true; }
dataLocationChangeListener.clear(); hostDomainPartition.setDeletable(true); WaitUntil.orDie(() -> !dataLocationChangeListener.isCalled()); assertFalse(dataLocationChangeListener.isCalled());
private static Set<Integer> getPartitionNumbers(Collection<HostDomainPartition> partition) { Set<Integer> partitionNumbers = Sets.newHashSet(); for (HostDomainPartition hostDomainPartition : partition) { partitionNumbers.add(hostDomainPartition.getPartitionNumber()); } return partitionNumbers; } }
@Override public HostDomainPartition getPartitionByNumber(int partNum) throws IOException { for (HostDomainPartition p : getPartitions()) { if (p.getPartitionNumber() == partNum) { return p; } } return null; }
@Override public int compareTo(HostDomainPartition o) { return Integer.valueOf(getPartitionNumber()).compareTo(o.getPartitionNumber()); }
public String getDataDirectory() { return assignment.getDisk(partition.getPartitionNumber()); }
private void deletePartition(HostDomain hostDomain, HostDomainPartition partition) throws IOException { LOG.info("Deleting Domain " + hostDomain.getDomain().getName() + " partition " + partition.getPartitionNumber()); Deleter deleter = hostDomain.getDomain().getStorageEngine().getDeleter(assignment, partition.getPartitionNumber()); deleter.delete(); hostDomain.removePartition(partition.getPartitionNumber()); }