/** * Returns a one to one replication map with consecutive shard Ids from a list of hosts. This method is useful for * quickly having an instance of ReplicationMap that has no replication (1 shard -> 1 host). */ public static ReplicationMap oneToOneMap(String... hosts) { List<ReplicationEntry> replicationMap = new ArrayList<ReplicationEntry>(); int shard = 0; for(String host : hosts) { ReplicationEntry entry = new ReplicationEntry(); entry.setNodes(Arrays.asList(new String[] { host })); entry.setShard(shard); replicationMap.add(entry); shard++; } return new ReplicationMap(replicationMap); }
while (repIter.hasNext()) { ReplicationEntry entry = repIter.next(); int partition = entry.getShard(); if (entry.getNodes().contains(dNodeInfo.getAddress())) { entry.getNodes().remove(dNodeInfo.getAddress()); if (entry.getNodes().isEmpty()) { repIter.remove(); pEntry.setShard(entry.getShard()); tablespace.getPartitionMap().getPartitionEntries().remove(pEntry); if (!entry.getNodes().isEmpty()) { nonEmptyReplicas++; ReplicationEntry reEntry = new ReplicationEntry(); reEntry.setShard(shard); reEntry.setExpectedReplicationFactor(metadata.getNReplicas()); reEntry.setNodes(new ArrayList<String>()); existingEntry.getNodes().remove(dNodeInfo.getAddress()); if (existingEntry.getNodes().isEmpty()) { replicationMap.remove(existingEntry); if (existingPartitionEntry != null) { if (!existingEntry.getNodes().contains(dNodeInfo.getAddress())) { existingEntry.getNodes().add(dNodeInfo.getAddress()); } else {
log.info(rEntry); if (rEntry.getNodes().size() > 0) { for (int i = rEntry.getNodes().size(); i < rEntry.getExpectedReplicationFactor(); i++) { String originNode = rEntry.getNodes().get(0); int roundRobinIndex = (roundRobinCount + k) % aliveDNodes.size(); String finalNode = aliveDNodes.get(roundRobinIndex); if (rEntry.getNodes().contains(finalNode)) { continue; .getTablespace(), rEntry.getShard(), entry.getKey().getVersion())); roundRobinCount++; break;
PartitionEntry pEntry = null; for (PartitionEntry partEntry : req.getPartitionMap()) { if (partEntry.getShard().equals(rEntry.getShard())) { pEntry = partEntry; String msg = "No Partition metadata for shard: " + rEntry.getShard() + " this is very likely to be a software bug."; log.error(msg); for (int i = 0; i < rEntry.getNodes().size(); i++) { String dnodeId = rEntry.getNodes().get(i); if (dnodeId.startsWith("tcp://")) { dnodeId = dnodeId.substring("tcp://".length(), dnodeId.length()); rEntry.getNodes().set(i, dnodeId); for (String dNode : rEntry.getNodes()) { List<DeployAction> actionsSoFar = (List<DeployAction>) MapUtils.getObject(actions, dNode, new ArrayList<DeployAction>()); actions.put(dNode, actionsSoFar); DeployAction deployAction = new DeployAction(); deployAction.setDataURI(req.getData_uri() + "/" + rEntry.getShard() + ".db"); deployAction.setTablespace(req.getTablespace()); deployAction.setVersion(version); deployAction.setPartition(rEntry.getShard()); metadata.setMinKey(pEntry.getMin()); metadata.setMaxKey(pEntry.getMax()); metadata.setNReplicas(rEntry.getNodes().size());
/** * Return true if one or more of the DNodes reported an error. */ private boolean checkForFailure() { IMap<String, String> deployErrorPanel = context.getCoordinationStructures().getDeployErrorPanel( version); if (!isReplicaBalancingEnabled) { return !deployErrorPanel.isEmpty(); } // If replica balancing is enabled we check whether we could survive after the failed DNodes Set<String> failedDNodes = new HashSet<String>(deployErrorPanel.keySet()); // Check if deploy needs to be canceled or if the system could auto-rebalance itself afterwards for (DeployRequest deployRequest : deployRequests) { for (ReplicationEntry repEntry : deployRequest.getReplicationMap()) { if (failedDNodes.containsAll(repEntry.getNodes())) { // There is AT LEAST one partition that depends on the failed DNodes so the deploy must fail! return true; } } } return false; } } /* End ManageDeploy */
if (rEntry.getShard() == partitionId) { repEntry = rEntry; if (repEntry.getNodes().size() == 0) { // No one alive for serving the int index = lastNode % repEntry.getNodes().size(); electedNode = repEntry.getNodes().get(index); partitionRoundRobin.get().put(partitionId, index); if (tried == repEntry.getNodes().size()) { return new ErrorQueryStatus("Error connecting dnode[" + electedNode + "] for " + msg); } else { log.warn("TTransportException problem when connecting dnode[" + electedNode + "] at trial[" + tried + "] of[" + repEntry.getNodes().size() + "] DNodes. Will retry. Info: " + msg, e); if (tried == repEntry.getNodes().size()) { return new ErrorQueryStatus("DNode exception [" + e.getMsg() + "] from dnode[" + electedNode + "] for " + msg); } else { log.warn("Error resolving query with dnode[" + electedNode + "] at trial[" + tried + "] of[" + repEntry.getNodes().size() + "] DNodes. Will retry. Info: " + msg, e); if (tried == repEntry.getNodes().size()) { return new ErrorQueryStatus("Error connecting dnode[" + electedNode + "] for " + msg); } else { log.warn("TException problem when connecting dnode[" + electedNode + "] at trial[" + tried + "] of[" + repEntry.getNodes().size() + "] DNodes. Will retry. Info: " + msg, e); if (tried == repEntry.getNodes().size()) { return new ErrorQueryStatus("Error creating pool for dnode[" + electedNode + "] for " + msg);
/** * Returns a one to one replication map with consecutive shard Ids from a list of hosts. This method is useful for * quickly having an instance of ReplicationMap that has no replication (1 shard -> 1 host). */ public static ReplicationMap oneToOneMap(String... hosts) { List<ReplicationEntry> replicationMap = new ArrayList<ReplicationEntry>(); int shard = 0; for (String host : hosts) { ReplicationEntry entry = new ReplicationEntry(); entry.setNodes(Arrays.asList(new String[]{host})); entry.setShard(shard); replicationMap.add(entry); shard++; } return new ReplicationMap(replicationMap); }
public static ReplicationMap roundRobinMap(int nPartitions, int repFactor, String... hosts) { List<ReplicationEntry> repEntries = new ArrayList<ReplicationEntry>(); for(int i = 0; i < nPartitions; i++) { int toNode = i % hosts.length; ReplicationEntry repEntry = new ReplicationEntry(); repEntry.setShard(i); List<String> shardDNodes = new ArrayList<String>(); for(int j = 0; j < repFactor; j++) { int chosen = (toNode + j) % hosts.length; shardDNodes.add(hosts[chosen]); } repEntry.setNodes(shardDNodes); repEntries.add(repEntry); } return new ReplicationMap(repEntries); }
public static ReplicationMap roundRobinMap(int nPartitions, int repFactor, String... hosts) { List<ReplicationEntry> repEntries = new ArrayList<ReplicationEntry>(); for (int i = 0; i < nPartitions; i++) { int toNode = i % hosts.length; ReplicationEntry repEntry = new ReplicationEntry(); repEntry.setShard(i); List<String> shardDNodes = new ArrayList<String>(); for (int j = 0; j < repFactor; j++) { int chosen = (toNode + j) % hosts.length; shardDNodes.add(hosts[chosen]); } repEntry.setNodes(shardDNodes); repEntries.add(repEntry); } return new ReplicationMap(repEntries); }