new ReplicationTarget(queueKey.substring(index + 1, secondIndex), queueKey.substring(secondIndex + 1, thirdIndex), Table.ID.of(queueKey.substring(thirdIndex + 1))));
/** * Deserialize a ReplicationTarget * * @param t * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(Text t) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(t.getBytes(), t.getLength()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
/** * Deserialize a ReplicationTarget * * @param s * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(String s) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(s.getBytes(UTF_8), s.length()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
public Set<ReplicationTarget> getReplicationTargets() { // The total set of configured targets final Set<ReplicationTarget> allConfiguredTargets = new HashSet<>(); final Map<String,Table.ID> tableNameToId = Tables.getNameToIdMap(context); for (String table : tableNameToId.keySet()) { if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) { continue; } Table.ID localId = tableNameToId.get(table); if (localId == null) { log.trace("Could not determine ID for {}", table); continue; } TableConfiguration tableConf = context.getServerConfFactory().getTableConfiguration(localId); if (tableConf == null) { log.trace("Could not get configuration for table {} (it no longer exists)", table); continue; } for (Entry<String,String> prop : tableConf .getAllPropertiesWithPrefix(Property.TABLE_REPLICATION_TARGET).entrySet()) { String peerName = prop.getKey() .substring(Property.TABLE_REPLICATION_TARGET.getKey().length()); String remoteIdentifier = prop.getValue(); ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId); allConfiguredTargets.add(target); } } return allConfiguredTargets; }
String peerName = prop.getKey().substring(targetPrefix.length()); String remoteIdentifier = prop.getValue(); ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId);
Mutation m = new Mutation(file); ReplicationTarget target = new ReplicationTarget(); DataOutputBuffer buffer = new DataOutputBuffer(); Text t = new Text();
new ReplicationTarget(queueKey.substring(index + 1, secondIndex), queueKey.substring(secondIndex + 1, thirdIndex), queueKey.substring(thirdIndex + 1)));
/** * Deserialize a ReplicationTarget * * @param t * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(Text t) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(t.getBytes(), t.getLength()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
/** * Deserialize a ReplicationTarget * * @param s * Serialized copy * @return the deserialized version */ public static ReplicationTarget from(String s) { ReplicationTarget target = new ReplicationTarget(); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(s.getBytes(UTF_8), s.length()); try { target.readFields(buffer); } catch (IOException e) { throw new RuntimeException(e); } return target; }
public Set<ReplicationTarget> getReplicationTargets() { // The total set of configured targets final Set<ReplicationTarget> allConfiguredTargets = new HashSet<>(); final Map<String,String> tableNameToId = Tables.getNameToIdMap(context.getInstance()); for (String table : tableNameToId.keySet()) { if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) { continue; } String localId = tableNameToId.get(table); if (null == localId) { log.trace("Could not determine ID for {}", table); continue; } TableConfiguration tableConf = context.getServerConfigurationFactory() .getTableConfiguration(localId); if (null == tableConf) { log.trace("Could not get configuration for table {} (it no longer exists)", table); continue; } for (Entry<String,String> prop : tableConf .getAllPropertiesWithPrefix(Property.TABLE_REPLICATION_TARGET).entrySet()) { String peerName = prop.getKey() .substring(Property.TABLE_REPLICATION_TARGET.getKey().length()); String remoteIdentifier = prop.getValue(); ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId); allConfiguredTargets.add(target); } } return allConfiguredTargets; }
@Test public void doNotCreateWorkForFilesNotNeedingIt() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"), target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString(); String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, StatusUtil.fileCreatedValue(5)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, StatusUtil.fileCreatedValue(10)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); HashSet<String> queuedWork = new HashSet<>(); assigner.setQueuedWork(queuedWork); assigner.setMaxQueueSize(Integer.MAX_VALUE); replay(workQueue); assigner.createWork(); verify(workQueue); }
Mutation m = new Mutation(file); ReplicationTarget target = new ReplicationTarget(); DataOutputBuffer buffer = new DataOutputBuffer(); Text t = new Text();
Set<ReplicationTarget> expectedTargets = new HashSet<>(); for (Entry<String,String> cluster : targetClusters.entrySet()) { expectedTargets.add(new ReplicationTarget(cluster.getKey(), cluster.getValue(), tableId));
ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); String serializedTarget = target.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target.getRemoteIdentifier()
ReplicationTarget expected = new ReplicationTarget("remote_cluster_1", "4", tableId); workMaker.setBatchWriter(bw); workMaker.addWorkRecord(new Text(file), StatusUtil.fileCreatedValue(timeCreated),
@Test public void recordsWithProgressUpdateBothTables() throws Exception { conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target = new ReplicationTarget("peer", "table1", "1"); // Create a single work record for a file to some peer BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat)); bw.addMutation(m); bw.close(); updater.run(); Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME); Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId()); // We should only rely on the correct begin attribute being returned Status actual = Status.parseFrom(entry.getValue().get()); Assert.assertEquals(stat.getBegin(), actual.getBegin()); }
Status stat3 = Status.newBuilder().setBegin(500).setEnd(1000).setClosed(true) .setInfiniteEnd(true).build(); ReplicationTarget target1 = new ReplicationTarget("peer1", "table1", "1"); ReplicationTarget target2 = new ReplicationTarget("peer2", "table2", "1"); ReplicationTarget target3 = new ReplicationTarget("peer3", "table3", "1");
@Test public void reprocessingOfCompletedWorkRemovesWork() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText();
Status stat3 = Status.newBuilder().setBegin(1).setEnd(1000).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target1 = new ReplicationTarget("peer1", "table1", "1"); ReplicationTarget target2 = new ReplicationTarget("peer2", "table2", "1"); ReplicationTarget target3 = new ReplicationTarget("peer3", "table3", "1");
@Test public void createWorkForFilesInCorrectOrder() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText();