Refine search
scope=scope.substring(1); scope = NodeBase.normalize(scope); int excludedCountInScope = 0; // the number of nodes in both scope & excludedNodes int excludedCountOffScope = 0; // the number of nodes outside scope & excludedNodes if (excludedNodes != null) { for (Node node : excludedNodes) { node = getNode(NodeBase.getPath(node)); if (node == null) { continue; if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR) .startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) { excludedCountInScope++; Node n = getNode(scope); int scopeNodeCount = 0; if (n != null) {
Node node = getNode(scope); if (!(node instanceof InnerNode)) { return excludedNodes != null && excludedNodes.contains(node) ? node = null; } else { node = getNode(excludedScope); if (!(node instanceof InnerNode)) { numOfDatanodes -= 1; availableNodes = countNumOfAvailableNodes(scope, excludedNodes); } else { availableNodes = countNumOfAvailableNodes("~" + excludedScope, excludedNodes); Node ret = null; if (availableNodes > 0) { ret = chooseRandom(innerNode, node, excludedNodes, numOfDatanodes, availableNodes);
/** convert a network tree to a string. */ @Override public String toString() { // print the number of racks StringBuilder tree = new StringBuilder(); tree.append("Number of racks: "); tree.append(numOfRacks); tree.append("\n"); // print the number of leaves int numOfLeaves = getNumOfLeaves(); tree.append("Expected number of leaves:"); tree.append(numOfLeaves); tree.append("\n"); // print nodes for(int i=0; i<numOfLeaves; i++) { tree.append(NodeBase.getPath(clusterMap.getLeaf(i, null))); tree.append("\n"); } return tree.toString(); }
@Override protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) { int clusterSize = clusterMap.getNumOfLeaves(); int totalNumOfReplicas = numOfChosen + numOfReplicas; if (totalNumOfReplicas > clusterSize) { numOfReplicas -= (totalNumOfReplicas-clusterSize); totalNumOfReplicas = clusterSize; } // No calculation needed when there is only one rack or picking one node. int numOfRacks = clusterMap.getNumOfRacks(); if (numOfRacks == 1 || totalNumOfReplicas <= 1) { return new int[] {numOfReplicas, totalNumOfReplicas}; } // If more racks than replicas, put one replica per rack. if (totalNumOfReplicas < numOfRacks) { return new int[] {numOfReplicas, 1}; } // If more replicas than racks, evenly spread the replicas. // This calculation rounds up. int maxNodesPerRack = (totalNumOfReplicas - 1) / numOfRacks + 1; return new int[] {numOfReplicas, maxNodesPerRack}; }
private Node addHostToNodeMapping(String host, String networkLoc) { Node node; if ((node = clusterMap.getNode(networkLoc+"/"+host)) == null) { node = new NodeBase(host, networkLoc); clusterMap.add(node); if (node.getLevel() < getNumTaskCacheLevels()) { LOG.fatal("Got a host whose level is: " + node.getLevel() + "." + " Should get at least a level of value: " + getNumTaskCacheLevels()); try { stopTracker(); } catch (IOException ie) { LOG.warn("Exception encountered during shutdown: " + StringUtils.stringifyException(ie)); System.exit(-1); } } hostnameToNodeMap.put(host, node); // Make an entry for the node at the max level in the cache nodesAtMaxLevel.add(getParentNode(node, getNumTaskCacheLevels() - 1)); } return node; }
public void testRemove() throws Exception { for(int i=0; i<dataNodes.length; i++) { cluster.remove(dataNodes[i]); } for(int i=0; i<dataNodes.length; i++) { assertFalse(cluster.contains(dataNodes[i])); } assertEquals(0, cluster.getNumOfLeaves()); for(int i=0; i<dataNodes.length; i++) { cluster.add(dataNodes[i]); } }
public void testCountNumNodes() throws Exception { NetworkTopology cluster = new NetworkTopology(); cluster.add(getNewNode("node1", "/d1/r1")); NodeElement node2 = getNewNode("node2", "/d1/r2"); cluster.add(node2); cluster.add(getNewNode("node3", "/d1/r3")); NodeElement node3 = getNewNode("node4", "/d1/r4"); cluster.add(node3); cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); NodeElement deadNode = getNewNode("node5", "/d1/r2"); excludedNodes.add(deadNode); assertEquals("4 nodes should be available with extra excluded Node", 4, cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); assertEquals("excluded nodes without ~ scope should be considered", 2, cluster.countNumOfAvailableNodes("~" + deadNode.getNetworkLocation(), excludedNodes)); assertEquals("excluded nodes with rack scope should be considered", 1, cluster.countNumOfAvailableNodes(deadNode.getNetworkLocation(), excludedNodes)); cluster.countNumOfAvailableNodes("~" + deadNode.getNetworkLocation(), excludedNodes)); cluster.countNumOfAvailableNodes("/non-exist", excludedNodes));
private void resetData() { this.cluster = new NetworkTopology(); this.overUtilizedDatanodes.clear(); this.aboveAvgUtilizedDatanodes.clear(); this.belowAvgUtilizedDatanodes.clear(); this.underUtilizedDatanodes.clear(); this.datanodes.clear(); this.sources.clear(); this.targets.clear(); this.avgUtilization = 0.0D; cleanGlobalBlockList(); }
/** Add a datanode. */ void addDatanode(final DatanodeDescriptor node) { // To keep host2DatanodeMap consistent with datanodeMap, // remove from host2DatanodeMap the datanodeDescriptor removed // from datanodeMap before adding node to host2DatanodeMap. synchronized(this) { host2DatanodeMap.remove(datanodeMap.put(node.getDatanodeUuid(), node)); } networktopology.add(node); // may throw InvalidTopologyException host2DatanodeMap.add(node); checkIfClusterIsNowMultiRack(node); resolveUpgradeDomain(node); if (LOG.isDebugEnabled()) { LOG.debug(getClass().getSimpleName() + ".addDatanode: " + "node " + node + " is added to datanodeMap."); } }
/** * Return a reference to the node given its string representation. * Default implementation delegates to {@link #getNode(String)}. * * <p>To be overridden in subclasses for specific NetworkTopology * implementations, as alternative to overriding the full {@link #add(Node)} * method. * * @param node The string representation of this node's network location is * used to retrieve a Node object. * @return a reference to the node; null if the node is not in the tree * * @see #add(Node) * @see #getNode(String) */ protected Node getNodeForNetworkLocation(Node node) { return getNode(node.getNetworkLocation()); }
/** * Randomly choose a node. * * @param scope range of nodes from which a node will be chosen * @return the chosen node * * @see #chooseRandom(String, Collection) */ public Node chooseRandom(final String scope) { return chooseRandom(scope, null); }
/** Remove a node * Update node counter and rack counter if necessary * @param node node to be removed; can be null */ public void remove(Node node) { if (node==null) return; if( node instanceof InnerNode ) { throw new IllegalArgumentException( "Not allow to remove an inner node: "+NodeBase.getPath(node)); } LOG.info("Removing a node: "+NodeBase.getPath(node)); netlock.writeLock().lock(); try { if (clusterMap.remove(node)) { InnerNode rack = (InnerNode)getNode(node.getNetworkLocation()); if (rack == null) { numOfRacks--; } } LOG.debug("NetworkTopology became:\n{}", this); } finally { netlock.writeLock().unlock(); } }
if (writer == null || !clusterMap.contains(writer)) { writer = storages[0].getDatanodeDescriptor(); int shortestDistance = clusterMap.getDistance(writer, shortestStorage.getDatanodeDescriptor()); int shortestIndex = index; for(int i = index + 1; i < storages.length; i++) { int currentDistance = clusterMap.getDistance(writer, storages[i].getDatanodeDescriptor()); if (shortestDistance>currentDistance) {
private DatanodeDescriptor[] chooseRandom(int numOfReplicas, String nodes, List<Node> excludedNodes) { List<DatanodeDescriptor> results = new ArrayList<DatanodeDescriptor>(); int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(nodes, excludedNodes); numOfReplicas = (numOfAvailableNodes<numOfReplicas)? numOfAvailableNodes:numOfReplicas; while(numOfReplicas > 0) { DatanodeDescriptor choosenNode = (DatanodeDescriptor)(clusterMap.chooseRandom(nodes)); if (!excludedNodes.contains(choosenNode)) { results.add(choosenNode); excludedNodes.add(choosenNode); numOfReplicas--; } } return (DatanodeDescriptor[])results.toArray( new DatanodeDescriptor[results.size()]); }
@Override public boolean match(NetworkTopology cluster, Node left, Node right) { return cluster.isOnSameRack(left, right); }
throws NotEnoughReplicasException { int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes( scope, excludedNodes); int refreshCounter = numOfAvailableNodes; if (excludedNodes.add(chosenNode)) { //was not in the excluded list if (LOG.isDebugEnabled() && builder != null) { builder.append("\nNode ").append(NodeBase.getPath(chosenNode)).append(" ["); numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(scope, excludedNodes); refreshCounter = numOfAvailableNodes;
@Test public void testCreateInvalidTopology() throws Exception { NetworkTopology invalCluster = new NetworkTopology(); DatanodeDescriptor invalDataNodes[] = new DatanodeDescriptor[] { DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1"), DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1"), DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1") }; invalCluster.add(invalDataNodes[0]); invalCluster.add(invalDataNodes[1]); try { invalCluster.add(invalDataNodes[2]); fail("expected InvalidTopologyException"); } catch (NetworkTopology.InvalidTopologyException e) { assertTrue(e.getMessage().startsWith("Failed to add ")); assertTrue(e.getMessage().contains( "You cannot have a rack and a non-rack node at the same " + "level of the network topology.")); } }
List<Node> exlcNodes, long blocksize) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { return new DatanodeDescriptor[0]; int clusterSize = clusterMap.getNumOfLeaves(); int totalNumOfReplicas = chosenNodes.size()+numOfReplicas; if (totalNumOfReplicas > clusterSize) { (totalNumOfReplicas-1)/clusterMap.getNumOfRacks()+2; if (!clusterMap.contains(writer)) { writer=null; clusterMap.getPipeline((writer == null) ? localNode : writer, pipeline); return pipeline;
final BlockStoragePolicy storagePolicy, EnumSet<AddBlockFlag> addBlockFlags) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { return DatanodeStorageInfo.EMPTY_ARRAY;