.fromQueueKey(work); String filename = entry.getKey(); String peerName = entry.getValue().getPeerName();
/** * Distribute the work for the given path with filename * * @param path * Path to the file being replicated * @param target * Target for the file to be replicated to */ @Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); if (queuedWork.contains(queueKey)) { log.debug("{} is already queued to be replicated to {}, not re-queueing", path, target); return false; } try { log.debug("Queued work for {} and {}", queueKey, path); workQueue.addWork(queueKey, path.toString()); queuedWork.add(queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {}", path, e); return false; } return true; }
@Override public void process(String workID, byte[] data) { ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID) .getValue(); String file = new String(data, UTF_8);
@Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); Map<Table.ID,String> workForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (workForPeer == null) { workForPeer = new HashMap<>(); this.queuedWorkByPeerName.put(target.getPeerName(), workForPeer); } String queuedWork = workForPeer.get(target.getSourceTableId()); if (queuedWork == null) { try { workQueue.addWork(queueKey, path.toString()); workForPeer.put(target.getSourceTableId(), queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {} to {}", path, target, e); return false; } return true; } else if (queuedWork.startsWith(path.getName())) { log.debug("Not re-queueing work for {} as it has already been queued for replication to {}", path, target); return false; } else { log.debug("Not queueing {} for work as {} must be replicated to {} first", path, queuedWork, target.getPeerName()); return false; } }
.fromQueueKey(work); String filename = entry.getKey(); String peerName = entry.getValue().getPeerName();
String key = DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename, target);
@Override public void process(String workID, byte[] data) { ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID) .getValue(); String file = new String(data, UTF_8);
/** * Distribute the work for the given path with filename * * @param path * Path to the file being replicated * @param target * Target for the file to be replicated to */ @Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); if (queuedWork.contains(queueKey)) { log.debug("{} is already queued to be replicated to {}, not re-queueing", path, target); return false; } try { log.debug("Queued work for {} and {}", queueKey, path); workQueue.addWork(queueKey, path.toString()); queuedWork.add(queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {}", path, e); return false; } return true; }
for (String queueKey : workQueue.getWorkQueued()) { Entry<String,ReplicationTarget> queueKeyPair = DistributedWorkQueueWorkAssignerHelper .fromQueueKey(queueKey); String filename = queueKeyPair.getKey(); ReplicationTarget target = queueKeyPair.getValue();
@Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); Map<String,String> workForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (null == workForPeer) { workForPeer = new HashMap<>(); this.queuedWorkByPeerName.put(target.getPeerName(), workForPeer); } String queuedWork = workForPeer.get(target.getSourceTableId()); if (null == queuedWork) { try { workQueue.addWork(queueKey, path.toString()); workForPeer.put(target.getSourceTableId(), queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {} to {}", path, target, e); return false; } return true; } else if (queuedWork.startsWith(path.getName())) { log.debug("Not re-queueing work for {} as it has already been queued for replication to {}", path, target); return false; } else { log.debug("Not queueing {} for work as {} must be replicated to {} first", path, queuedWork, target.getPeerName()); return false; } }
String key = DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename, target);
workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1); expectLastCall().once(); workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2); expectLastCall().once(); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId())); Assert.assertEquals(1, cluster2Work.size()); Assert.assertTrue(cluster2Work.containsKey(target2.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster2Work.get(target2.getSourceTableId()));
Map<String,String> queuedWorkForCluster = new HashMap<>(); queuedWorkForCluster.put(target.getSourceTableId(), DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target)); queuedWork.put("cluster1", queuedWorkForCluster); workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target), file2); expectLastCall().once(); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target), cluster1Work.get(target.getSourceTableId()));
workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1); expectLastCall().once(); workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2); expectLastCall().once(); Assert.assertEquals(2, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster1Work.get(target2.getSourceTableId()));
workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), file1); expectLastCall().once(); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), cluster1Work.get(target.getSourceTableId()));