@Override public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException { String peerNode = getHFileRefsPeerNode(peerId); LOG.debug("Adding hfile references {} in queue {}", pairs, peerNode); List<ZKUtilOp> listOfOps = pairs.stream().map(p -> p.getSecond().getName()) .map(n -> getHFileNode(peerNode, n)) .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList()); LOG.debug("The multi list size for adding hfile references in zk for node {} is {}", peerNode, listOfOps.size()); try { ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e); } }
private void addLastSeqIdsToOps(String queueId, Map<String, Long> lastSeqIds, List<ZKUtilOp> listOfOps) throws KeeperException, ReplicationException { String peerId = new ReplicationQueueInfo(queueId).getPeerId(); for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) { String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId); Pair<Long, Integer> p = getLastSequenceIdWithVersion(lastSeqEntry.getKey(), peerId); byte[] data = ZKUtil.positionToByteArray(lastSeqEntry.getValue()); if (p.getSecond() < 0) { // ZNode does not exist. ZKUtil.createWithParents(zookeeper, path.substring(0, path.lastIndexOf(ZNodePaths.ZNODE_PATH_SEPARATOR))); listOfOps.add(ZKUtilOp.createAndFailSilent(path, data)); continue; } // Perform CAS in a specific version v0 (HBASE-20138) int v0 = p.getSecond(); long lastPushedSeqId = p.getFirst(); if (lastSeqEntry.getValue() <= lastPushedSeqId) { continue; } listOfOps.add(ZKUtilOp.setData(path, data, v0)); } }
ZKUtil.createAndFailSilent(watcher, znode); zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode, ProtobufUtil.prependPBMagic(proto.toByteArray())));
private void addLastSeqIdsToOps(String queueId, Map<String, Long> lastSeqIds, List<ZKUtilOp> listOfOps) throws KeeperException, ReplicationException { String peerId = new ReplicationQueueInfo(queueId).getPeerId(); for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) { String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId); Pair<Long, Integer> p = getLastSequenceIdWithVersion(lastSeqEntry.getKey(), peerId); byte[] data = ZKUtil.positionToByteArray(lastSeqEntry.getValue()); if (p.getSecond() < 0) { // ZNode does not exist. ZKUtil.createWithParents(zookeeper, path.substring(0, path.lastIndexOf(ZNodePaths.ZNODE_PATH_SEPARATOR))); listOfOps.add(ZKUtilOp.createAndFailSilent(path, data)); continue; } // Perform CAS in a specific version v0 (HBASE-20138) int v0 = p.getSecond(); long lastPushedSeqId = p.getFirst(); if (lastSeqEntry.getValue() <= lastPushedSeqId) { continue; } listOfOps.add(ZKUtilOp.setData(path, data, v0)); } }
@Test public void testSingleFailureInMulti() throws Exception { // try a multi where all but one operation succeeds String pathA = ZNodePaths.joinZNode(zkw.znodePaths.baseZNode, "testSingleFailureInMultiA"); String pathB = ZNodePaths.joinZNode(zkw.znodePaths.baseZNode, "testSingleFailureInMultiB"); String pathC = ZNodePaths.joinZNode(zkw.znodePaths.baseZNode, "testSingleFailureInMultiC"); LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA))); ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB))); ops.add(ZKUtilOp.deleteNodeFailSilent(pathC)); boolean caughtNoNode = false; try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; } assertTrue(caughtNoNode); // assert that none of the operations succeeded assertTrue(ZKUtil.checkExists(zkw, pathA) == -1); assertTrue(ZKUtil.checkExists(zkw, pathB) == -1); assertTrue(ZKUtil.checkExists(zkw, pathC) == -1); }
@Test public void testSingleFailureInMulti() throws Exception { // try a multi where all but one operation succeeds String pathA = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiA"); String pathB = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiB"); String pathC = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiC"); LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA))); ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB))); ops.add(ZKUtilOp.deleteNodeFailSilent(pathC)); boolean caughtNoNode = false; try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; } assertTrue(caughtNoNode); // assert that none of the operations succeeded assertTrue(ZKUtil.checkExists(zkw, pathA) == -1); assertTrue(ZKUtil.checkExists(zkw, pathB) == -1); assertTrue(ZKUtil.checkExists(zkw, pathC) == -1); }
@Override public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled) throws ReplicationException { try { ZKUtil.createWithParents(zookeeper, peersZNode); ZKUtil.multiOrSequential(zookeeper, Arrays.asList( ZKUtilOp.createAndFailSilent(getPeerNode(peerId), ReplicationPeerConfigUtil.toByteArray(peerConfig)), ZKUtilOp.createAndFailSilent(getPeerStateNode(peerId), enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES)), false); } catch (KeeperException e) { throw new ReplicationException("Could not add peer with id=" + peerId + ", peerConfif=>" + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"), e); } }
@Override public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException { String peerNode = getHFileRefsPeerNode(peerId); LOG.debug("Adding hfile references {} in queue {}", pairs, peerNode); List<ZKUtilOp> listOfOps = pairs.stream().map(p -> p.getSecond().getName()) .map(n -> getHFileNode(peerNode, n)) .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList()); LOG.debug("The multi list size for adding hfile references in zk for node {} is {}", peerNode, listOfOps.size()); try { ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e); } }
/** * Creates the specified node containing specified data, iff the node does not exist. Does * not set a watch and fails silently if the node already exists. * * The node created is persistent and open access. * * @param zkw zk reference * @param znode path of node * @param data a byte array data to store in the znode * @throws KeeperException if unexpected zookeeper exception */ public static void createAndFailSilent(ZKWatcher zkw, String znode, byte[] data) throws KeeperException { createAndFailSilent(zkw, (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data)); }
/** * Creates the specified node containing specified data, iff the node does not exist. Does * not set a watch and fails silently if the node already exists. * * The node created is persistent and open access. * * @param zkw zk reference * @param znode path of node * @param data a byte array data to store in the znode * @throws KeeperException if unexpected zookeeper exception */ public static void createAndFailSilent(ZooKeeperWatcher zkw, String znode, byte[] data) throws KeeperException { createAndFailSilent(zkw, (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data)); }
/** * Creates the specified node containing specified data, iff the node does not exist. Does * not set a watch and fails silently if the node already exists. * * The node created is persistent and open access. * * @param zkw zk reference * @param znode path of node * @param data a byte array data to store in the znode * @throws KeeperException if unexpected zookeeper exception */ public static void createAndFailSilent(ZooKeeperWatcher zkw, String znode, byte[] data) throws KeeperException { createAndFailSilent(zkw, (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data)); }
SortedSet<String> logQueue = new TreeSet<>(); listOfOps.add(ZKUtilOp.createAndFailSilent(newQueueNode, HConstants.EMPTY_BYTE_ARRAY)); LOG.debug("Creating {} with data {}", wal, Bytes.toStringBinary(logOffset)); String newWalNode = getFileNode(newQueueNode, wal); listOfOps.add(ZKUtilOp.createAndFailSilent(newWalNode, logOffset)); listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalNode)); logQueue.add(wal);
ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); ZKUtil.multiOrSequential(zkw, ops, false); String pathW = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureW"); ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- already exists ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathV))); // pass ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathW))); // pass boolean caughtNodeExists = false; try { ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- exists boolean caughtNoNode = false; try {
create4Nodes.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1))); create4Nodes.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2))); create4Nodes.add(ZKUtilOp.createAndFailSilent(path3, Bytes.toBytes(path3))); create4Nodes.add(ZKUtilOp.createAndFailSilent(path4, Bytes.toBytes(path4))); ZKUtil.multiOrSequential(zkw, create4Nodes, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.toBytes(path1))); ops.add(ZKUtilOp.deleteNodeFailSilent(path4)); ops.add(ZKUtilOp.createAndFailSilent(path5, Bytes.toBytes(path5))); ops.add(ZKUtilOp.createAndFailSilent(path6, Bytes.toBytes(path6))); ZKUtil.multiOrSequential(zkw, ops, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1),
ops.add(ZKUtilOp.createAndFailSilent(path, Bytes.toBytes(path))); ZKUtil.multiOrSequential(zkw, ops, false); try {
@Test public void testSimpleMulti() throws Exception { // null multi ZKUtil.multiOrSequential(zkw, null, false); // empty multi ZKUtil.multiOrSequential(zkw, new LinkedList<>(), false); // single create String path = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSimpleMulti"); LinkedList<ZKUtilOp> singleCreate = new LinkedList<>(); singleCreate.add(ZKUtilOp.createAndFailSilent(path, new byte[0])); ZKUtil.multiOrSequential(zkw, singleCreate, false); assertTrue(ZKUtil.checkExists(zkw, path) != -1); // single setdata LinkedList<ZKUtilOp> singleSetData = new LinkedList<>(); byte [] data = Bytes.toBytes("foobar"); singleSetData.add(ZKUtilOp.setData(path, data)); ZKUtil.multiOrSequential(zkw, singleSetData, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path), data)); // single delete LinkedList<ZKUtilOp> singleDelete = new LinkedList<>(); singleDelete.add(ZKUtilOp.deleteNodeFailSilent(path)); ZKUtil.multiOrSequential(zkw, singleDelete, false); assertTrue(ZKUtil.checkExists(zkw, path) == -1); }
@Test public void testRunSequentialOnMultiFailure() throws Exception { String path1 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential1"); String path2 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential2"); String path3 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential3"); String path4 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential4"); // create some nodes that we will use later LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1))); ops.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2))); ZKUtil.multiOrSequential(zkw, ops, false); // test that, even with operations that fail, the ones that would pass will pass // with runSequentialOnMultiFailure ops = new LinkedList<>(); ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); // pass ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); // pass ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); // fail -- node doesn't exist ops.add(ZKUtilOp.createAndFailSilent(path4, Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); // pass ZKUtil.multiOrSequential(zkw, ops, true); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); assertTrue(ZKUtil.checkExists(zkw, path2) == -1); assertTrue(ZKUtil.checkExists(zkw, path3) == -1); assertFalse(ZKUtil.checkExists(zkw, path4) == -1); }
@Override public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled, SyncReplicationState syncReplicationState) throws ReplicationException { List<ZKUtilOp> multiOps = Arrays.asList( ZKUtilOp.createAndFailSilent(getPeerNode(peerId), ReplicationPeerConfigUtil.toByteArray(peerConfig)), ZKUtilOp.createAndFailSilent(getPeerStateNode(peerId), enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES), ZKUtilOp.createAndFailSilent(getSyncReplicationStateNode(peerId), SyncReplicationState.toByteArray(syncReplicationState)), ZKUtilOp.createAndFailSilent(getNewSyncReplicationStateNode(peerId), NONE_STATE_ZNODE_BYTES)); try { ZKUtil.createWithParents(zookeeper, peersZNode); ZKUtil.multiOrSequential(zookeeper, multiOps, false); } catch (KeeperException e) { throw new ReplicationException( "Could not add peer with id=" + peerId + ", peerConfig=>" + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED") + ", syncReplicationState=" + syncReplicationState, e); } }
@Test public void testSingleFailureInMulti() throws Exception { // try a multi where all but one operation succeeds String pathA = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiA"); String pathB = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiB"); String pathC = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiC"); LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA))); ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB))); ops.add(ZKUtilOp.deleteNodeFailSilent(pathC)); boolean caughtNoNode = false; try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; } assertTrue(caughtNoNode); // assert that none of the operations succeeded assertTrue(ZKUtil.checkExists(zkw, pathA) == -1); assertTrue(ZKUtil.checkExists(zkw, pathB) == -1); assertTrue(ZKUtil.checkExists(zkw, pathC) == -1); }
/** * Creates the specified node containing specified data, iff the node does not exist. Does * not set a watch and fails silently if the node already exists. * * The node created is persistent and open access. * * @param zkw zk reference * @param znode path of node * @param data a byte array data to store in the znode * @throws KeeperException if unexpected zookeeper exception */ public static void createAndFailSilent(ZKWatcher zkw, String znode, byte[] data) throws KeeperException { createAndFailSilent(zkw, (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data)); }