/** * Adds work to the queue, automatically converting the String to bytes using UTF-8 */ public void addWork(String workId, String data) throws KeeperException, InterruptedException { addWork(workId, data.getBytes(UTF_8)); }
/** * Distribute the work for the given path with filename * * @param path * Path to the file being replicated * @param target * Target for the file to be replicated to */ @Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); if (queuedWork.contains(queueKey)) { log.debug("{} is already queued to be replicated to {}, not re-queueing", path, target); return false; } try { log.debug("Queued work for {} and {}", queueKey, path); workQueue.addWork(queueKey, path.toString()); queuedWork.add(queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {}", path, e); return false; } return true; }
private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException { String work = source + "|" + destination; new DistributedWorkQueue(master.getZooKeeperRoot() + Constants.ZRECOVERY, master.getConfiguration()).addWork(sortId, work.getBytes(UTF_8)); synchronized (this) { sortsQueued.add(sortId); } final String path = master.getZooKeeperRoot() + Constants.ZRECOVERY + "/" + sortId; log.info("Created zookeeper entry {} with data {}", path, work); }
@Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); Map<Table.ID,String> workForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (workForPeer == null) { workForPeer = new HashMap<>(); this.queuedWorkByPeerName.put(target.getPeerName(), workForPeer); } String queuedWork = workForPeer.get(target.getSourceTableId()); if (queuedWork == null) { try { workQueue.addWork(queueKey, path.toString()); workForPeer.put(target.getSourceTableId(), queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {} to {}", path, target, e); return false; } return true; } else if (queuedWork.startsWith(path.getName())) { log.debug("Not re-queueing work for {} as it has already been queued for replication to {}", path, target); return false; } else { log.debug("Not queueing {} for work as {} must be replicated to {} first", path, queuedWork, target.getPeerName()); return false; } }
continue; bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8)); workIds.add(orig.getName()); log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
/** * Adds work to the queue, automatically converting the String to bytes using UTF-8 */ public void addWork(String workId, String data) throws KeeperException, InterruptedException { addWork(workId, data.getBytes(UTF_8)); }
/** * Distribute the work for the given path with filename * * @param path * Path to the file being replicated * @param target * Target for the file to be replicated to */ @Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); if (queuedWork.contains(queueKey)) { log.debug("{} is already queued to be replicated to {}, not re-queueing", path, target); return false; } try { log.debug("Queued work for {} and {}", queueKey, path); workQueue.addWork(queueKey, path.toString()); queuedWork.add(queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {}", path, e); return false; } return true; }
private void initiateSort(String sortId, String source, final String destination, AccumuloConfiguration aconf) throws KeeperException, InterruptedException, IOException { String work = source + "|" + destination; new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY, aconf) .addWork(sortId, work.getBytes(UTF_8)); synchronized (this) { sortsQueued.add(sortId); } final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId; log.info("Created zookeeper entry " + path + " with data " + work); }
private void initiateSort(String host, final String file) throws KeeperException, InterruptedException { String source = getSource(host, file).toString(); new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(file, source.getBytes(UTF_8)); synchronized (this) { sortsQueued.add(file); } final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + file; log.info("Created zookeeper entry " + path + " with data " + source); }
@Override protected boolean queueWork(Path path, ReplicationTarget target) { String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.getName(), target); Map<String,String> workForPeer = this.queuedWorkByPeerName.get(target.getPeerName()); if (null == workForPeer) { workForPeer = new HashMap<>(); this.queuedWorkByPeerName.put(target.getPeerName(), workForPeer); } String queuedWork = workForPeer.get(target.getSourceTableId()); if (null == queuedWork) { try { workQueue.addWork(queueKey, path.toString()); workForPeer.put(target.getSourceTableId(), queueKey); } catch (KeeperException | InterruptedException e) { log.warn("Could not queue work for {} to {}", path, target, e); return false; } return true; } else if (queuedWork.startsWith(path.getName())) { log.debug("Not re-queueing work for {} as it has already been queued for replication to {}", path, target); return false; } else { log.debug("Not queueing {} for work as {} must be replicated to {} first", path, queuedWork, target.getPeerName()); return false; } }
continue; bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8)); workIds.add(orig.getName()); log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
continue; bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8)); workIds.add(orig.getName()); log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target), file2); expectLastCall().once();
workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), file1); expectLastCall().once();